diff --git a/.cargo/config b/.cargo/config deleted file mode 100644 index a4dc4b423fe..00000000000 --- a/.cargo/config +++ /dev/null @@ -1,13 +0,0 @@ -[target.x86_64-unknown-redox] -linker = "x86_64-unknown-redox-gcc" - -[target.'cfg(feature = "cargo-clippy")'] -rustflags = [ - "-Wclippy::use_self", - "-Wclippy::needless_pass_by_value", - "-Wclippy::semicolon_if_nothing_returned", - "-Wclippy::single_char_pattern", - "-Wclippy::explicit_iter_loop", - "-Wclippy::if_not_else", -] - diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000000..c6aa207614f --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +[target.x86_64-unknown-redox] +linker = "x86_64-unknown-redox-gcc" + +[env] +PROJECT_NAME_FOR_VERSION_STRING = "uutils coreutils" diff --git a/.clippy.toml b/.clippy.toml index 814e40b6960..113f003ad81 100644 --- a/.clippy.toml +++ b/.clippy.toml @@ -1,2 +1,4 @@ -msrv = "1.70.0" -cognitive-complexity-threshold = 10 +avoid-breaking-exported-api = false +check-private-items = true +cognitive-complexity-threshold = 24 +missing-docs-in-crate-items = true diff --git a/.config/nextest.toml b/.config/nextest.toml index 3ba8bb393a4..473c461402a 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -4,3 +4,10 @@ status-level = "all" final-status-level = "skip" failure-output = "immediate-final" fail-fast = false + +[profile.coverage] +retries = 0 +status-level = "all" +final-status-level = "skip" +failure-output = "immediate-final" +fail-fast = false diff --git a/.editorconfig b/.editorconfig index 53ccc4f9a15..9df8cbbbf98 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,4 +1,5 @@ # EditorConfig (is awesome!; ref: http://EditorConfig.org; v2022.02.11 [rivy]) +# spell-checker:ignore akefile shellcheck vcproj # * top-most EditorConfig file root = true @@ -52,7 +53,7 @@ indent_style = space switch_case_indent = true [*.{sln,vc{,x}proj{,.*},[Ss][Ln][Nn],[Vv][Cc]{,[Xx]}[Pp][Rr][Oo][Jj]{,.*}}] -# MSVC sln/vcproj/vcxproj files, when used, will persistantly revert to CRLF EOLNs and eat final EOLs +# MSVC sln/vcproj/vcxproj files, when used, will persistently revert to CRLF EOLNs and eat final EOLs end_of_line = crlf insert_final_newline = false diff --git a/.envrc b/.envrc new file mode 100644 index 00000000000..cbf4a76e2de --- /dev/null +++ b/.envrc @@ -0,0 +1,7 @@ +# spell-checker:ignore direnv + +if ! has nix_direnv_version || ! nix_direnv_version 3.0.6; then + source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.6/direnvrc" "sha256-RYcUJaRMf8oF5LznDrlCXbkOQrywm0HDv1VjYGaJGdM=" +fi + +use flake diff --git a/.github/workflows/CICD.yml b/.github/workflows/CICD.yml index ad12ede6833..80b2fa24053 100644 --- a/.github/workflows/CICD.yml +++ b/.github/workflows/CICD.yml @@ -2,24 +2,26 @@ name: CICD # spell-checker:ignore (abbrev/names) CICD CodeCOV MacOS MinGW MSVC musl taiki # spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic CARGOFLAGS -# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers -# spell-checker:ignore (people) Peltoche rivy dtolnay -# spell-checker:ignore (shell/tools) choco clippy dmake dpkg esac fakeroot fdesc fdescfs gmake grcov halium lcov libssl mkdir popd printf pushd rsync rustc rustfmt rustup shopt utmpdump xargs -# spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos gnueabihf issuecomment maint multisize nullglob onexitbegin onexitend pell runtest Swatinem tempfile testsuite toybox uutils +# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers dedupe devel profdata +# spell-checker:ignore (people) Peltoche rivy dtolnay Anson dawidd +# spell-checker:ignore (shell/tools) binutils choco clippy dmake dpkg esac fakeroot fdesc fdescfs gmake grcov halium lcov libclang libfuse libssl limactl mkdir nextest nocross pacman popd printf pushd redoxer rsync rustc rustfmt rustup shopt sccache utmpdump xargs +# spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos getenforce gnueabihf issuecomment maint manpages msys multisize noconfirm nullglob onexitbegin onexitend pell runtest Swatinem tempfile testsuite toybox uutils env: PROJECT_NAME: coreutils PROJECT_DESC: "Core universal (cross-platform) utilities" PROJECT_AUTH: "uutils" - RUST_MIN_SRV: "1.70.0" + RUST_MIN_SRV: "1.85.0" # * style job configuration STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis on: pull_request: push: + tags: + - '*' branches: - - main + - '*' permissions: contents: read # to fetch code (actions/checkout) @@ -35,7 +37,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: EmbarkStudios/cargo-deny-action@v1 + with: + persist-credentials: false + - uses: EmbarkStudios/cargo-deny-action@v2 style_deps: ## ToDO: [2021-11-10; rivy] 'Style/deps' needs more informative output and better integration of results into the GHA dashboard @@ -52,6 +56,8 @@ jobs: - { os: windows-latest , features: feat_os_windows } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@nightly ## note: requires 'nightly' toolchain b/c `cargo-udeps` uses the `rustc` '-Z save-analysis' option ## * ... ref: @@ -104,13 +110,19 @@ jobs: # - { os: windows-latest , features: feat_os_windows } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: stable components: clippy - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install/setup prerequisites + shell: bash + run: | + sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - name: Initialize workflow variables id: vars shell: bash @@ -137,7 +149,7 @@ jobs: shell: bash run: | RUSTDOCFLAGS="-Dwarnings" cargo doc ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --no-deps --workspace --document-private-items - - uses: DavidAnson/markdownlint-cli2-action@v15 + - uses: DavidAnson/markdownlint-cli2-action@v19 with: fix: "true" globs: | @@ -157,6 +169,8 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.RUST_MIN_SRV }} @@ -164,7 +178,7 @@ jobs: - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Initialize workflow variables id: vars shell: bash @@ -176,19 +190,19 @@ jobs: unset CARGO_FEATURES_OPTION if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features "${{ matrix.job.features }}"' ; fi outputs CARGO_FEATURES_OPTION - - name: Confirm MinSRV compatible 'Cargo.lock' + - name: Confirm MinSRV compatible '*/Cargo.lock' shell: bash run: | - ## Confirm MinSRV compatible 'Cargo.lock' - # * 'Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) - cargo fetch --locked --quiet || { echo "::error file=Cargo.lock::Incompatible (or out-of-date) 'Cargo.lock' file; update using \`cargo +${{ env.RUST_MIN_SRV }} update\`" ; exit 1 ; } - - name: Confirm MinSRV equivalence for '.clippy.toml' + ## Confirm MinSRV compatible '*/Cargo.lock' + # * '*/Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) + for dir in "." "fuzz"; do + ( cd "$dir" && cargo fetch --locked --quiet ) || { echo "::error file=$dir/Cargo.lock::Incompatible (or out-of-date) '$dir/Cargo.lock' file; update using \`cd '$dir' && cargo +${{ env.RUST_MIN_SRV }} update\`" ; exit 1 ; } + done + - name: Install/setup prerequisites shell: bash run: | - ## Confirm MinSRV equivalence for '.clippy.toml' - # * ensure '.clippy.toml' MSRV configuration setting is equal to ${{ env.RUST_MIN_SRV }} - CLIPPY_MSRV=$(grep -P "(?i)^\s*msrv\s*=\s*" .clippy.toml | grep -oP "\d+([.]\d+)+") - if [ "${CLIPPY_MSRV}" != "${{ env.RUST_MIN_SRV }}" ]; then { echo "::error file=.clippy.toml::Incorrect MSRV configuration for clippy (found '${CLIPPY_MSRV}'; should be '${{ env.RUST_MIN_SRV }}'); update '.clippy.toml' with 'msrv = \"${{ env.RUST_MIN_SRV }}\"'" ; exit 1 ; } ; fi + # Install a package for one of the tests + sudo apt-get -y update ; sudo apt-get -y install attr - name: Info shell: bash run: | @@ -225,6 +239,8 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - name: "`cargo update` testing" @@ -232,7 +248,9 @@ jobs: run: | ## `cargo update` testing # * convert any errors/warnings to GHA UI annotations; ref: - cargo fetch --locked --quiet || { echo "::error file=Cargo.lock::'Cargo.lock' file requires update (use \`cargo +${{ env.RUST_MIN_SRV }} update\`)" ; exit 1 ; } + for dir in "." "fuzz"; do + ( cd "$dir" && cargo fetch --locked --quiet ) || { echo "::error file=$dir/Cargo.lock::'$dir/Cargo.lock' file requires update (use \`cd '$dir' && cargo +${{ env.RUST_MIN_SRV }} update\`)" ; exit 1 ; } + done build_makefile: name: Build/Makefile @@ -248,11 +266,17 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 + - name: Install/setup prerequisites + shell: bash + run: | + sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: "`make build`" shell: bash run: | @@ -262,10 +286,26 @@ jobs: run: make nextest CARGOFLAGS="--profile ci --hide-progress-bar" env: RUST_BACKTRACE: "1" + - name: "`make install COMPLETIONS=n MANPAGES=n`" + shell: bash + run: | + DESTDIR=/tmp/ make PROFILE=release COMPLETIONS=n MANPAGES=n install + # Check that the utils are present + test -f /tmp/usr/local/bin/tty + # Check that the manpage is not present + ! test -f /tmp/usr/local/share/man/man1/whoami.1 + # Check that the completion is not present + ! test -f /tmp/usr/local/share/zsh/site-functions/_install + ! test -f /tmp/usr/local/share/bash-completion/completions/head + ! test -f /tmp/usr/local/share/fish/vendor_completions.d/cat.fish + env: + RUST_BACKTRACE: "1" - name: "`make install`" shell: bash run: | DESTDIR=/tmp/ make PROFILE=release install + # Check that the utils are present + test -f /tmp/usr/local/bin/tty # Check that the manpage is present test -f /tmp/usr/local/share/man/man1/whoami.1 # Check that the completion is present @@ -278,6 +318,8 @@ jobs: shell: bash run: | DESTDIR=/tmp/ make uninstall + # Check that the utils are not present + ! test -f /tmp/usr/local/bin/tty # Check that the manpage is not present ! test -f /tmp/usr/local/share/man/man1/whoami.1 # Check that the completion is not present @@ -302,13 +344,15 @@ jobs: - { os: windows-latest , features: feat_os_windows } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Test - run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + run: cargo nextest run --hide-progress-bar --profile ci --features ${{ matrix.job.features }} env: RUST_BACKTRACE: "1" @@ -329,13 +373,15 @@ jobs: - { os: windows-latest , features: feat_os_windows } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@nightly - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Test - run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + run: cargo nextest run --hide-progress-bar --profile ci --features ${{ matrix.job.features }} env: RUST_BACKTRACE: "1" @@ -353,16 +399,18 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Install dependencies shell: bash run: | ## Install dependencies sudo apt-get update - sudo apt-get install jq + sudo apt-get install jq libselinux1-dev - name: "`make install`" shell: bash run: | @@ -395,14 +443,14 @@ jobs: --arg multisize "$SIZE_MULTI" \ '{($date): { sha: $sha, size: $size, multisize: $multisize, }}' > size-result.json - name: Download the previous individual size result - uses: dawidd6/action-download-artifact@v3 + uses: dawidd6/action-download-artifact@v9 with: workflow: CICD.yml name: individual-size-result repo: uutils/coreutils path: dl - name: Download the previous size result - uses: dawidd6/action-download-artifact@v3 + uses: dawidd6/action-download-artifact@v9 with: workflow: CICD.yml name: size-result @@ -465,23 +513,29 @@ jobs: fail-fast: false matrix: job: - # - { os , target , cargo-options , features , use-cross , toolchain, skip-tests } - - { os: ubuntu-latest , target: arm-unknown-linux-gnueabihf, features: feat_os_unix_gnueabihf, use-cross: use-cross, } - - { os: ubuntu-latest , target: aarch64-unknown-linux-gnu , features: feat_os_unix_gnueabihf , use-cross: use-cross } - - { os: ubuntu-latest , target: aarch64-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } + # - { os , target , cargo-options , default-features, features , use-cross , toolchain, skip-tests, workspace-tests, skip-package, skip-publish } + - { os: ubuntu-latest , target: arm-unknown-linux-gnueabihf , features: feat_os_unix_gnueabihf , use-cross: use-cross , skip-tests: true } + - { os: ubuntu-24.04-arm , target: aarch64-unknown-linux-gnu , features: feat_os_unix_gnueabihf } + - { os: ubuntu-latest , target: aarch64-unknown-linux-musl , features: feat_os_unix , use-cross: use-cross , skip-tests: true } # - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: feat_selinux , use-cross: use-cross } - - { os: ubuntu-latest , target: i686-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } - - { os: ubuntu-latest , target: i686-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } - - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } - - { os: ubuntu-latest , target: x86_64-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } - - { os: ubuntu-latest , target: x86_64-unknown-redox , features: feat_os_unix_redox , use-cross: redoxer , skip-tests: true } - - { os: macos-latest , target: aarch64-apple-darwin , features: feat_os_macos , use-cross: use-cross, skip-tests: true} # Hopefully github provides free M1 runners soon... - - { os: macos-latest , target: x86_64-apple-darwin , features: feat_os_macos } + - { os: ubuntu-latest , target: i686-unknown-linux-gnu , features: "feat_os_unix,test_risky_names", use-cross: use-cross } + - { os: ubuntu-latest , target: i686-unknown-linux-musl , features: feat_os_unix , use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: "feat_os_unix,test_risky_names", use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: "feat_os_unix,uudoc" , use-cross: no, workspace-tests: true } + - { os: ubuntu-latest , target: x86_64-unknown-linux-musl , features: feat_os_unix , use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-redox , features: feat_os_unix_redox , use-cross: redoxer , skip-tests: true } + - { os: ubuntu-latest , target: wasm32-unknown-unknown , default-features: false, features: uucore/format, skip-tests: true, skip-package: true, skip-publish: true } + - { os: macos-latest , target: aarch64-apple-darwin , features: feat_os_macos, workspace-tests: true } # M1 CPU + - { os: macos-13 , target: x86_64-apple-darwin , features: feat_os_macos, workspace-tests: true } - { os: windows-latest , target: i686-pc-windows-msvc , features: feat_os_windows } - - { os: windows-latest , target: x86_64-pc-windows-gnu , features: feat_os_windows } + # TODO: Re-enable after rust-onig release: https://github.com/rust-onig/rust-onig/issues/193 + # - { os: windows-latest , target: x86_64-pc-windows-gnu , features: feat_os_windows } - { os: windows-latest , target: x86_64-pc-windows-msvc , features: feat_os_windows } + - { os: windows-latest , target: aarch64-pc-windows-msvc , features: feat_os_windows, use-cross: use-cross , skip-tests: true } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.RUST_MIN_SRV }} @@ -490,7 +544,7 @@ jobs: with: key: "${{ matrix.job.os }}_${{ matrix.job.target }}" - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Initialize workflow variables id: vars shell: bash @@ -567,12 +621,19 @@ jobs: CARGO_FEATURES_OPTION='' ; if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features=${{ matrix.job.features }}' ; fi outputs CARGO_FEATURES_OPTION + # * CARGO_DEFAULT_FEATURES_OPTION + CARGO_DEFAULT_FEATURES_OPTION='' ; + if [ "${{ matrix.job.default-features }}" == "false" ]; then CARGO_DEFAULT_FEATURES_OPTION='--no-default-features' ; fi + outputs CARGO_DEFAULT_FEATURES_OPTION # * CARGO_CMD CARGO_CMD='cross' CARGO_CMD_OPTIONS='+${{ env.RUST_MIN_SRV }}' + # Added suffix for artifacts, needed when multiple jobs use the same target. + ARTIFACTS_SUFFIX='' case '${{ matrix.job.use-cross }}' in ''|0|f|false|n|no) CARGO_CMD='cargo' + ARTIFACTS_SUFFIX='-nocross' ;; redoxer) CARGO_CMD='redoxer' @@ -581,13 +642,22 @@ jobs: esac outputs CARGO_CMD outputs CARGO_CMD_OPTIONS + outputs ARTIFACTS_SUFFIX + CARGO_TEST_OPTIONS='' + case '${{ matrix.job.workspace-tests }}' in + 1|t|true|y|yes) + # This also runs tests in other packages in the source directory (e.g. uucore). + # We cannot enable this everywhere as some platforms are currently broken, and + # we cannot use `cross` as its Docker image is ancient (Ubuntu 16.04) and is + # missing required system dependencies (e.g. recent libclang-dev). + CARGO_TEST_OPTIONS='--workspace' + ;; + esac + outputs CARGO_TEST_OPTIONS # ** pass needed environment into `cross` container (iff `cross` not already configured via "Cross.toml") if [ "${CARGO_CMD}" = 'cross' ] && [ ! -e "Cross.toml" ] ; then printf "[build.env]\npassthrough = [\"CI\", \"RUST_BACKTRACE\", \"CARGO_TERM_COLOR\"]\n" > Cross.toml fi - # * test only library and/or binaries for arm-type targets - unset CARGO_TEST_OPTIONS ; case '${{ matrix.job.target }}' in aarch64-* | arm-*) CARGO_TEST_OPTIONS="--bins" ;; esac; - outputs CARGO_TEST_OPTIONS # * executable for `strip`? STRIP="strip" case ${{ matrix.job.target }} in @@ -612,21 +682,21 @@ jobs: run: | ## Install/setup prerequisites case '${{ matrix.job.target }}' in - arm-unknown-linux-gnueabihf) + arm-unknown-linux-gnueabihf) sudo apt-get -y update sudo apt-get -y install gcc-arm-linux-gnueabihf ;; - aarch64-unknown-linux-*) + aarch64-unknown-linux-*) sudo apt-get -y update sudo apt-get -y install gcc-aarch64-linux-gnu ;; - *-redox*) + *-redox*) sudo apt-get -y update sudo apt-get -y install fuse3 libfuse-dev ;; # Update binutils if MinGW due to https://github.com/rust-lang/rust/issues/112368 x86_64-pc-windows-gnu) - C:/msys64/usr/bin/pacman.exe -Syu --needed mingw-w64-x86_64-gcc --noconfirm + C:/msys64/usr/bin/pacman.exe -Sy --needed mingw-w64-x86_64-gcc --noconfirm echo "C:\msys64\mingw64\bin" >> $GITHUB_PATH ;; esac @@ -635,6 +705,9 @@ jobs: esac case '${{ matrix.job.os }}' in ubuntu-*) + # selinux headers needed to build tests + sudo apt-get -y update + sudo apt-get -y install libselinux1-dev # pinky is a tool to show logged-in users from utmp, and gecos fields from /etc/passwd. # In GitHub Action *nix VMs, no accounts log in, even the "runner" account that runs the commands. The account also has empty gecos fields. # To work around this for pinky tests, we create a fake login entry for the GH runner account... @@ -685,20 +758,20 @@ jobs: # dependencies echo "## dependency list" cargo fetch --locked --quiet - cargo tree --locked --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --no-dedupe -e=no-dev --prefix=none | grep -vE "$PWD" | sort --unique + cargo tree --locked --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} --no-dedupe -e=no-dev --prefix=none | grep -vE "$PWD" | sort --unique - name: Build shell: bash run: | ## Build ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} build --release \ - --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} - name: Test if: matrix.job.skip-tests != true shell: bash run: | ## Test ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} test --target=${{ matrix.job.target }} \ - ${{ steps.vars.outputs.CARGO_TEST_OPTIONS}} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + ${{ steps.vars.outputs.CARGO_TEST_OPTIONS}} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} env: RUST_BACKTRACE: "1" - name: Test individual utilities @@ -707,15 +780,16 @@ jobs: run: | ## Test individual utilities ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} test --target=${{ matrix.job.target }} \ - ${{ steps.vars.outputs.CARGO_TEST_OPTIONS}} ${{ matrix.job.cargo-options }} ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} + ${{ matrix.job.cargo-options }} ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} env: RUST_BACKTRACE: "1" - name: Archive executable artifacts uses: actions/upload-artifact@v4 with: - name: ${{ env.PROJECT_NAME }}-${{ matrix.job.target }} + name: ${{ env.PROJECT_NAME }}-${{ matrix.job.target }}${{ steps.vars.outputs.ARTIFACTS_SUFFIX }} path: target/${{ matrix.job.target }}/release/${{ env.PROJECT_NAME }}${{ steps.vars.outputs.EXE_suffix }} - name: Package + if: matrix.job.skip-package != true shell: bash run: | ## Package artifact(s) @@ -750,9 +824,10 @@ jobs: fakeroot dpkg-deb --build "${DPKG_DIR}" "${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.DPKG_NAME }}" fi - name: Publish - uses: softprops/action-gh-release@v1 - if: steps.vars.outputs.DEPLOY + uses: softprops/action-gh-release@v2 + if: steps.vars.outputs.DEPLOY && matrix.job.skip-publish != true with: + draft: true files: | ${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_NAME }} ${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.DPKG_NAME }} @@ -779,12 +854,15 @@ jobs: ## VARs setup echo "TEST_SUMMARY_FILE=busybox-result.json" >> $GITHUB_OUTPUT - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Install/setup prerequisites shell: bash run: | + sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev ## Install/setup prerequisites make prepare-busytest - name: Run BusyBox test suite @@ -859,23 +937,28 @@ jobs: TEST_SUMMARY_FILE="toybox-result.json" outputs TEST_SUMMARY_FILE - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.RUST_MIN_SRV }} components: rustfmt - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install/setup prerequisites + shell: bash + run: | + sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - name: Build coreutils as multiple binaries shell: bash run: | ## Build individual uutil binaries set -v make - - name: Install/setup prerequisites + - name: Run toybox src shell: bash run: | - ## Install/setup prerequisites make toybox-src - name: Run Toybox test suite id: summary @@ -937,45 +1020,55 @@ jobs: matrix: job: - { os: ubuntu-latest , features: unix, toolchain: nightly } - - { os: macos-latest , features: macos, toolchain: nightly } - - { os: windows-latest , features: windows, toolchain: nightly-x86_64-pc-windows-gnu } + # FIXME: Re-enable macos code coverage + # - { os: macos-latest , features: macos, toolchain: nightly } + # FIXME: Re-enable Code Coverage on windows, which currently fails due to "profiler_builtins". See #6686. + # - { os: windows-latest , features: windows, toolchain: nightly-x86_64-pc-windows-gnu } steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.job.toolchain }} components: rustfmt - - uses: taiki-e/install-action@nextest - - uses: taiki-e/install-action@grcov + - uses: taiki-e/install-action@v2 + with: + tool: nextest,grcov@0.8.24 - uses: Swatinem/rust-cache@v2 + - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 + # - name: Reattach HEAD ## may be needed for accurate code coverage info # run: git checkout ${{ github.head_ref }} + - name: Initialize workflow variables id: vars shell: bash run: | ## VARs setup outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + # toolchain TOOLCHAIN="nightly" ## default to "nightly" toolchain (required for certain required unstable compiler flags) ## !maint: refactor when stable channel has needed support + # * specify gnu-type TOOLCHAIN for windows; `grcov` requires gnu-style code coverage data files case ${{ matrix.job.os }} in windows-*) TOOLCHAIN="$TOOLCHAIN-x86_64-pc-windows-gnu" ;; esac; + # * use requested TOOLCHAIN if specified if [ -n "${{ matrix.job.toolchain }}" ]; then TOOLCHAIN="${{ matrix.job.toolchain }}" ; fi outputs TOOLCHAIN - # staging directory - STAGING='_staging' - outputs STAGING + # target-specific options + # * CARGO_FEATURES_OPTION CARGO_FEATURES_OPTION='--all-features' ; ## default to '--all-features' for code coverage if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features=${{ matrix.job.features }}' ; fi outputs CARGO_FEATURES_OPTION + # * CODECOV_FLAGS CODECOV_FLAGS=$( echo "${{ matrix.job.os }}" | sed 's/[^[:alnum:]]/_/g' ) outputs CODECOV_FLAGS + - name: Install/setup prerequisites shell: bash run: | @@ -983,6 +1076,7 @@ jobs: case '${{ matrix.job.os }}' in macos-latest) brew install coreutils ;; # needed for testing esac + case '${{ matrix.job.os }}' in ubuntu-latest) # pinky is a tool to show logged-in users from utmp, and gecos fields from /etc/passwd. @@ -998,58 +1092,118 @@ jobs: echo "foo" > /home/runner/.plan ;; esac + case '${{ matrix.job.os }}' in # Update binutils if MinGW due to https://github.com/rust-lang/rust/issues/112368 - windows-latest) C:/msys64/usr/bin/pacman.exe -Syu --needed mingw-w64-x86_64-gcc --noconfirm ; echo "C:\msys64\mingw64\bin" >> $GITHUB_PATH ;; + windows-latest) C:/msys64/usr/bin/pacman.exe -Sy --needed mingw-w64-x86_64-gcc --noconfirm ; echo "C:\msys64\mingw64\bin" >> $GITHUB_PATH ;; esac - - name: Initialize toolchain-dependent workflow variables - id: dep_vars - shell: bash + + ## Install the llvm-tools component to get access to `llvm-profdata` + rustup component add llvm-tools + + - name: Run test and coverage + id: run_test_cov run: | - ## Dependent VARs setup outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # * determine sub-crate utility list - UTILITY_LIST="$(./util/show-utils.sh ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }})" - CARGO_UTILITY_LIST_OPTIONS="$(for u in ${UTILITY_LIST}; do echo -n "-puu_${u} "; done;)" - outputs CARGO_UTILITY_LIST_OPTIONS - - name: Test - run: cargo nextest run --profile ci --hide-progress-bar ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} -p uucore -p coreutils - env: - RUSTC_WRAPPER: "" - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - RUSTDOCFLAGS: "-Cpanic=abort" - RUST_BACKTRACE: "1" - # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }} - - name: Test individual utilities - run: cargo nextest run --profile ci --hide-progress-bar ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} + + # Run the coverage script + ./util/build-run-test-coverage-linux.sh + + outputs REPORT_FILE env: - RUSTC_WRAPPER: "" - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - RUSTDOCFLAGS: "-Cpanic=abort" - RUST_BACKTRACE: "1" + COVERAGE_DIR: ${{ github.workspace }}/coverage + FEATURES_OPTION: ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }} - - name: Generate coverage data (via `grcov`) - id: coverage - shell: bash - run: | - ## Generate coverage data - COVERAGE_REPORT_DIR="target/debug" - COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" - # GRCOV_IGNORE_OPTION='--ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*"' ## `grcov` ignores these params when passed as an environment variable (why?) - # GRCOV_EXCLUDE_OPTION='--excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()"' ## `grcov` ignores these params when passed as an environment variable (why?) - mkdir -p "${COVERAGE_REPORT_DIR}" - # display coverage files - ~/.cargo/bin/grcov . --output-type files --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique - # generate coverage report - ~/.cargo/bin/grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" - echo "report=${COVERAGE_REPORT_FILE}" >> $GITHUB_OUTPUT + - name: Upload coverage results (to Codecov.io) - uses: codecov/codecov-action@v3 - # if: steps.vars.outputs.HAS_CODECOV_TOKEN + uses: codecov/codecov-action@v5 with: - # token: ${{ secrets.CODECOV_TOKEN }} - file: ${{ steps.coverage.outputs.report }} + token: ${{ secrets.CODECOV_TOKEN }} + file: ${{ steps.run_test_cov.outputs.report }} ## flags: IntegrationTests, UnitTests, ${{ steps.vars.outputs.CODECOV_FLAGS }} flags: ${{ steps.vars.outputs.CODECOV_FLAGS }} name: codecov-umbrella fail_ci_if_error: false + + test_separately: + name: Separate Builds + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: build and test all programs individually + shell: bash + run: | + for f in $(util/show-utils.sh) + do + echo "Building and testing $f" + cargo test -p "uu_$f" || exit 1 + done + + test_all_features: + name: Test all features separately + needs: [ min_version, deps ] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + # windows-latest - https://github.com/uutils/coreutils/issues/7044 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: build and test all features individually + shell: bash + run: | + for f in $(util/show-utils.sh) + do + echo "Running tests with --features=$f and --no-default-features" + cargo test --features=$f --no-default-features + done + + test_selinux: + name: Build/SELinux + needs: [ min_version, deps ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - name: Setup Lima + uses: lima-vm/lima-actions/setup@v1 + id: lima-actions-setup + - name: Cache ~/.cache/lima + uses: actions/cache@v4 + with: + path: ~/.cache/lima + key: lima-${{ steps.lima-actions-setup.outputs.version }} + - name: Start Fedora VM with SELinux + run: limactl start --plain --name=default --cpus=1 --disk=30 --memory=4 --network=lima:user-v2 template://fedora + - name: Setup SSH + uses: lima-vm/lima-actions/ssh@v1 + - run: rsync -v -a -e ssh . lima-default:~/work/ + - name: Setup Rust and other build deps in VM + run: | + lima sudo dnf install gcc g++ git rustup libselinux-devel clang-devel attr -y + lima rustup-init -y --default-toolchain stable + - name: Verify SELinux Status + run: | + lima getenforce + lima ls -laZ /etc/selinux + - name: Build and Test with SELinux + run: | + lima ls + lima bash -c "cd work && cargo test --features 'feat_selinux'" + - name: Lint with SELinux + run: lima bash -c "cd work && cargo clippy --all-targets --features 'feat_selinux' -- -D warnings" diff --git a/.github/workflows/CheckScripts.yml b/.github/workflows/CheckScripts.yml index c18c4733cbe..78a4656fcde 100644 --- a/.github/workflows/CheckScripts.yml +++ b/.github/workflows/CheckScripts.yml @@ -1,6 +1,6 @@ name: CheckScripts -# spell-checker:ignore ludeeus mfinelli +# spell-checker:ignore ludeeus mfinelli shellcheck scandir shfmt env: SCRIPT_DIR: 'util' @@ -8,7 +8,7 @@ env: on: push: branches: - - main + - '*' paths: - 'util/**/*.sh' pull_request: @@ -30,6 +30,8 @@ jobs: contents: read steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Run ShellCheck uses: ludeeus/action-shellcheck@master env: @@ -46,6 +48,8 @@ jobs: contents: read steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Setup shfmt uses: mfinelli/setup-shfmt@v3 - name: Run shfmt diff --git a/.github/workflows/FixPR.yml b/.github/workflows/FixPR.yml index e837b354687..bbdf50b30b5 100644 --- a/.github/workflows/FixPR.yml +++ b/.github/workflows/FixPR.yml @@ -1,6 +1,6 @@ name: FixPR -# spell-checker:ignore Swatinem dtolnay +# spell-checker:ignore Swatinem dtolnay dedupe # Trigger automated fixes for PRs being merged (with associated commits) @@ -27,6 +27,8 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Initialize job variables id: vars shell: bash @@ -41,9 +43,11 @@ jobs: - name: Ensure updated 'Cargo.lock' shell: bash run: | - # Ensure updated 'Cargo.lock' - # * 'Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) - cargo fetch --locked --quiet || cargo +${{ steps.vars.outputs.RUST_MIN_SRV }} update + # Ensure updated '*/Cargo.lock' + # * '*/Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) + for dir in "." "fuzz"; do + ( cd "$dir" && (cargo fetch --locked --quiet || cargo +${{ steps.vars.outputs.RUST_MIN_SRV }} update) ) + done - name: Info shell: bash run: | @@ -69,8 +73,8 @@ jobs: with: new_branch: ${{ env.BRANCH_TARGET }} default_author: github_actions - message: "maint ~ refresh 'Cargo.lock'" - add: Cargo.lock + message: "maint ~ refresh 'Cargo.lock' 'fuzz/Cargo.lock'" + add: Cargo.lock fuzz/Cargo.lock env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -86,6 +90,8 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Initialize job variables id: vars shell: bash diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml index 36c54490ce9..7fe42070e82 100644 --- a/.github/workflows/GnuComment.yml +++ b/.github/workflows/GnuComment.yml @@ -1,10 +1,12 @@ name: GnuComment +# spell-checker:ignore zizmor backquote + on: workflow_run: workflows: ["GnuTests"] types: - - completed + - completed # zizmor: ignore[dangerous-triggers] permissions: {} jobs: diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 89fa5a7da50..b12dbb235aa 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -1,8 +1,8 @@ name: GnuTests # spell-checker:ignore (abbrev/names) CodeCov gnulib GnuTests Swatinem -# spell-checker:ignore (jargon) submodules -# spell-checker:ignore (libs/utils) autopoint chksum gperf lcov libexpect pyinotify shopt texinfo valgrind libattr libcap taiki-e +# spell-checker:ignore (jargon) submodules devel +# spell-checker:ignore (libs/utils) autopoint chksum getenforce gperf lcov libexpect limactl pyinotify setenforce shopt texinfo valgrind libattr libcap taiki-e # spell-checker:ignore (options) Ccodegen Coverflow Cpanic Zpanic # spell-checker:ignore (people) Dawid Dziurla * dawidd dtolnay # spell-checker:ignore (vars) FILESET SUBDIRS XPASS @@ -13,7 +13,7 @@ on: pull_request: push: branches: - - main + - '*' permissions: contents: read @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} +env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + jobs: gnu: permissions: @@ -30,7 +33,7 @@ jobs: contents: read # for actions/checkout to fetch code pull-requests: read # for dawidd6/action-download-artifact to query commit hash name: Run GNU tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Initialize workflow variables id: vars @@ -45,23 +48,31 @@ jobs: path_reference="reference" outputs path_GNU path_GNU_tests path_reference path_UUTILS # - repo_default_branch="${{ github.event.repository.default_branch }}" - repo_GNU_ref="v9.4" - repo_reference_branch="${{ github.event.repository.default_branch }}" + repo_default_branch="$DEFAULT_BRANCH" + repo_GNU_ref="v9.7" + repo_reference_branch="$DEFAULT_BRANCH" outputs repo_default_branch repo_GNU_ref repo_reference_branch # SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log" ROOT_SUITE_LOG_FILE="${path_GNU_tests}/test-suite-root.log" + SELINUX_SUITE_LOG_FILE="${path_GNU_tests}/selinux-test-suite.log" + SELINUX_ROOT_SUITE_LOG_FILE="${path_GNU_tests}/selinux-test-suite-root.log" TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support TEST_FILESET_PREFIX='test-fileset-IDs.sha1#' TEST_FILESET_SUFFIX='.txt' TEST_SUMMARY_FILE='gnu-result.json' TEST_FULL_SUMMARY_FILE='gnu-full-result.json' - outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE + TEST_ROOT_FULL_SUMMARY_FILE='gnu-root-full-result.json' + TEST_SELINUX_FULL_SUMMARY_FILE='selinux-gnu-full-result.json' + TEST_SELINUX_ROOT_FULL_SUMMARY_FILE='selinux-root-gnu-full-result.json' + AGGREGATED_SUMMARY_FILE='aggregated-result.json' + + outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE SELINUX_SUITE_LOG_FILE SELINUX_ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE TEST_ROOT_FULL_SUMMARY_FILE TEST_SELINUX_FULL_SUMMARY_FILE TEST_SELINUX_ROOT_FULL_SUMMARY_FILE AGGREGATED_SUMMARY_FILE - name: Checkout code (uutil) uses: actions/checkout@v4 with: path: '${{ steps.vars.outputs.path_UUTILS }}' + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: stable @@ -75,9 +86,57 @@ jobs: repository: 'coreutils/coreutils' path: '${{ steps.vars.outputs.path_GNU }}' ref: ${{ steps.vars.outputs.repo_GNU_ref }} - submodules: recursive + submodules: false + persist-credentials: false + + - name: Selinux - Setup Lima + uses: lima-vm/lima-actions/setup@v1 + id: lima-actions-setup + + - name: Selinux - Cache ~/.cache/lima + uses: actions/cache@v4 + with: + path: ~/.cache/lima + key: lima-${{ steps.lima-actions-setup.outputs.version }} + + - name: Selinux - Start Fedora VM with SELinux + run: limactl start --plain --name=default --cpus=4 --disk=40 --memory=8 --network=lima:user-v2 template://fedora + + - name: Selinux - Setup SSH + uses: lima-vm/lima-actions/ssh@v1 + + - name: Selinux - Verify SELinux Status and Configuration + run: | + lima getenforce + lima ls -laZ /etc/selinux + lima sudo sestatus + + # Ensure we're running in enforcing mode + lima sudo setenforce 1 + lima getenforce + + # Create test files with SELinux contexts for testing + lima sudo mkdir -p /var/test_selinux + lima sudo touch /var/test_selinux/test_file + lima sudo chcon -t etc_t /var/test_selinux/test_file + lima ls -Z /var/test_selinux/test_file # Verify context + + - name: Selinux - Install dependencies in VM + run: | + lima sudo dnf -y update + lima sudo dnf -y install git autoconf autopoint bison texinfo gperf gcc g++ gdb jq libacl-devel libattr-devel libcap-devel libselinux-devel attr rustup clang-devel texinfo-tex wget automake patch quilt + lima rustup-init -y --default-toolchain stable + + - name: Override submodule URL and initialize submodules + # Use github instead of upstream git server + run: | + git submodule sync --recursive + git config submodule.gnulib.url https://github.com/coreutils/gnulib.git + git submodule update --init --recursive --depth 1 + working-directory: ${{ steps.vars.outputs.path_GNU }} + - name: Retrieve reference artifacts - uses: dawidd6/action-download-artifact@v3 + uses: dawidd6/action-download-artifact@v9 # ref: continue-on-error: true ## don't break the build for missing reference artifacts (may be expired or just not generated yet) with: @@ -91,7 +150,7 @@ jobs: run: | ## Install dependencies sudo apt-get update - sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev + sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev libselinux1-dev attr quilt - name: Add various locales shell: bash run: | @@ -111,12 +170,68 @@ jobs: sudo update-locale echo "After:" locale -a + + - name: Selinux - Copy the sources to VM + run: | + rsync -a -e ssh . lima-default:~/work/ + - name: Build binaries shell: bash run: | ## Build binaries cd '${{ steps.vars.outputs.path_UUTILS }}' bash util/build-gnu.sh --release-build + + - name: Selinux - Generate selinux tests list + run: | + # Find and list all tests that require SELinux + lima bash -c "cd ~/work/gnu/ && grep -l 'require_selinux_' -r tests/ > ~/work/uutils/selinux-tests.txt" + lima bash -c "cd ~/work/uutils/ && cat selinux-tests.txt" + + # Count the tests + lima bash -c "cd ~/work/uutils/ && echo 'Found SELinux tests:'; wc -l selinux-tests.txt" + + - name: Selinux - Build for selinux tests + run: | + lima bash -c "cd ~/work/uutils/ && bash util/build-gnu.sh" + lima bash -c "mkdir -p ~/work/gnu/tests-selinux/" + + - name: Selinux - Run selinux tests + run: | + lima sudo setenforce 1 + lima getenforce + lima cat /proc/filesystems + lima bash -c "cd ~/work/uutils/ && bash util/run-gnu-test.sh \$(cat selinux-tests.txt)" + + - name: Selinux - Extract testing info from individual logs into JSON + shell: bash + run : | + lima bash -c "cd ~/work/gnu/ && python3 ../uutils/util/gnu-json-result.py tests > ~/work/gnu/tests-selinux/${{ steps.vars.outputs.TEST_SELINUX_FULL_SUMMARY_FILE }}" + + - name: Selinux/root - Run selinux tests + run: | + lima bash -c "cd ~/work/uutils/ && CI=1 bash util/run-gnu-test.sh run-root \$(cat selinux-tests.txt)" + + - name: Selinux/root - Extract testing info from individual logs into JSON + shell: bash + run : | + lima bash -c "cd ~/work/gnu/ && python3 ../uutils/util/gnu-json-result.py tests > ~/work/gnu/tests-selinux/${{ steps.vars.outputs.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }}" + + - name: Selinux - Collect test logs and test results + run: | + mkdir -p ${{ steps.vars.outputs.path_GNU_tests }}-selinux + + # Copy the test logs from the Lima VM to the host + lima bash -c "cp ~/work/gnu/tests/test-suite.log ~/work/gnu/tests-selinux/ || echo 'No test-suite.log found'" + lima bash -c "cp ~/work/gnu/tests/test-suite-root.log ~/work/gnu/tests-selinux/ || echo 'No test-suite-root.log found'" + rsync -v -a -e ssh lima-default:~/work/gnu/tests-selinux/ ./${{ steps.vars.outputs.path_GNU_tests }}-selinux/ + + # Copy SELinux logs to the main test directory for integrated processing + cp -f ${{ steps.vars.outputs.path_GNU_tests }}-selinux/test-suite.log ${{ steps.vars.outputs.path_GNU_tests }}/selinux-test-suite.log + cp -f ${{ steps.vars.outputs.path_GNU_tests }}-selinux/test-suite-root.log ${{ steps.vars.outputs.path_GNU_tests }}/selinux-test-suite-root.log + cp -f ${{ steps.vars.outputs.path_GNU_tests }}-selinux/${{ steps.vars.outputs.TEST_SELINUX_FULL_SUMMARY_FILE }} . + cp -f ${{ steps.vars.outputs.path_GNU_tests }}-selinux/${{ steps.vars.outputs.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }} . + - name: Run GNU tests shell: bash run: | @@ -124,6 +239,13 @@ jobs: path_GNU='${{ steps.vars.outputs.path_GNU }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' bash "${path_UUTILS}/util/run-gnu-test.sh" + + - name: Extract testing info from individual logs into JSON + shell: bash + run : | + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' + python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} + - name: Run GNU root tests shell: bash run: | @@ -131,35 +253,40 @@ jobs: path_GNU='${{ steps.vars.outputs.path_GNU }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' bash "${path_UUTILS}/util/run-gnu-test.sh" run-root - - name: Extract testing info into JSON + + - name: Extract testing info from individual logs (run as root) into JSON shell: bash run : | - ## Extract testing info into JSON path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' - python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} + python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_ROOT_FULL_SUMMARY_FILE }} + - name: Extract/summarize testing info id: summary shell: bash run: | ## Extract/summarize testing info outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' - # - SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}' - ROOT_SUITE_LOG_FILE='${{ steps.vars.outputs.ROOT_SUITE_LOG_FILE }}' - ls -al ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} - if test -f "${SUITE_LOG_FILE}" + # Check if the file exists + if test -f "${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }}" then - source ${path_UUTILS}/util/analyze-gnu-results.sh ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} + # Look at all individual results and summarize + eval $(python3 ${path_UUTILS}/util/analyze-gnu-results.py -o=${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }} ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} ${{ steps.vars.outputs.TEST_ROOT_FULL_SUMMARY_FILE }} ${{ steps.vars.outputs.TEST_SELINUX_FULL_SUMMARY_FILE }} ${{ steps.vars.outputs.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }}) + if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then - echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early" + echo "::error ::Failed to parse test results from '${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }}'; failing early" exit 1 fi + output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP" echo "${output}" - if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi + + if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then + echo "::warning ::${output}" + fi + jq -n \ --arg date "$(date --rfc-email)" \ --arg sha "$GITHUB_SHA" \ @@ -173,9 +300,10 @@ jobs: HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) outputs HASH else - echo "::error ::Failed to find summary of test results (missing '${SUITE_LOG_FILE}'); failing early" + echo "::error ::Failed to find summary of test results (missing '${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }}'); failing early" exit 1 fi + # Compress logs before upload (fails otherwise) gzip ${{ steps.vars.outputs.TEST_LOGS_GLOB }} - name: Reserve SHA1/ID of 'test-summary' @@ -196,121 +324,74 @@ jobs: - name: Upload full json results uses: actions/upload-artifact@v4 with: - name: gnu-full-result.json + name: gnu-full-result path: ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} + - name: Upload root json results + uses: actions/upload-artifact@v4 + with: + name: gnu-root-full-result + path: ${{ steps.vars.outputs.TEST_ROOT_FULL_SUMMARY_FILE }} + - name: Upload selinux json results + uses: actions/upload-artifact@v4 + with: + name: selinux-gnu-full-result + path: ${{ steps.vars.outputs.TEST_SELINUX_FULL_SUMMARY_FILE }} + - name: Upload selinux root json results + uses: actions/upload-artifact@v4 + with: + name: selinux-root-gnu-full-result.json + path: ${{ steps.vars.outputs.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }} + - name: Upload aggregated json results + uses: actions/upload-artifact@v4 + with: + name: aggregated-result + path: ${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }} - name: Compare test failures VS reference shell: bash run: | - ## Compare test failures VS reference - have_new_failures="" - REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' - ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log' - REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' + ## Compare test failures VS reference using JSON files + REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/aggregated-result/aggregated-result.json' + CURRENT_SUMMARY_FILE='${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }}' REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' - # https://github.com/uutils/coreutils/issues/4294 - # https://github.com/uutils/coreutils/issues/4295 - IGNORE_INTERMITTENT="${path_UUTILS}/.github/workflows/ignore-intermittent.txt" - mkdir -p ${{ steps.vars.outputs.path_reference }} + # Path to ignore file for intermittent issues + IGNORE_INTERMITTENT="${path_UUTILS}/.github/workflows/ignore-intermittent.txt" + # Set up comment directory COMMENT_DIR="${{ steps.vars.outputs.path_reference }}/comment" mkdir -p ${COMMENT_DIR} echo ${{ github.event.number }} > ${COMMENT_DIR}/NR COMMENT_LOG="${COMMENT_DIR}/result.txt" - # The comment log might be downloaded from a previous run - # We only want the new changes, so remove it if it exists. - rm -f ${COMMENT_LOG} - touch ${COMMENT_LOG} - - compare_tests() { - local new_log_file=$1 - local ref_log_file=$2 - local test_type=$3 # "standard" or "root" - - if test -f "${ref_log_file}"; then - echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}" - REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) - CURRENT_RUN_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) - REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) - CURRENT_RUN_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) - echo "Detailled information:" - echo "REF_ERROR = ${REF_ERROR}" - echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}" - echo "REF_FAILING = ${REF_FAILING}" - echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}" - - # Compare failing and error tests - for LINE in ${CURRENT_RUN_FAILING} - do - if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" - then - if ! grep ${LINE} ${IGNORE_INTERMITTENT} - then - MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" - echo "::error ::$MSG" - echo $MSG >> ${COMMENT_LOG} - have_new_failures="true" - else - MSG="Skip an intermittent issue ${LINE} (fails in this run but passes in the 'main' branch)" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - echo "" - fi - fi - done - - for LINE in ${REF_FAILING} - do - if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_FAILING}" - then - if ! grep ${LINE} ${IGNORE_INTERMITTENT} - then - MSG="Congrats! The gnu test ${LINE} is no longer failing!" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - else - MSG="Skipping an intermittent issue ${LINE} (passes in this run but fails in the 'main' branch)" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - echo "" - fi - fi - done - - for LINE in ${CURRENT_RUN_ERROR} - do - if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" - then - MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" - echo "::error ::$MSG" - echo $MSG >> ${COMMENT_LOG} - have_new_failures="true" - fi - done - - for LINE in ${REF_ERROR} - do - if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_ERROR}" - then - MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - fi - done - else - echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available." - fi - } + COMPARISON_RESULT=0 + if test -f "${CURRENT_SUMMARY_FILE}"; then + if test -f "${REF_SUMMARY_FILE}"; then + echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")" - # Compare standard tests - compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard" + python3 ${path_UUTILS}/util/compare_test_results.py \ + --ignore-file "${IGNORE_INTERMITTENT}" \ + --output "${COMMENT_LOG}" \ + "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" - # Compare root tests - compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root" + COMPARISON_RESULT=$? + else + echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'." + fi + else + echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early" + exit 1 + fi - if test -n "${have_new_failures}" ; then exit -1 ; fi + if [ ${COMPARISON_RESULT} -eq 1 ]; then + echo "ONLY_INTERMITTENT=false" >> $GITHUB_ENV + echo "::error ::Found new non-intermittent test failures" + exit 1 + else + echo "ONLY_INTERMITTENT=true" >> $GITHUB_ENV + echo "::notice ::No new test failures detected" + fi - name: Upload comparison log (for GnuComment workflow) if: success() || failure() # run regardless of prior step success/failure uses: actions/upload-artifact@v4 @@ -330,76 +411,3 @@ jobs: else echo "::warning ::Skipping test summary comparison; no prior reference summary is available." fi - - gnu_coverage: - name: Run GNU tests with coverage - runs-on: ubuntu-22.04 - steps: - - name: Checkout code uutil - uses: actions/checkout@v4 - with: - path: 'uutils' - - name: Checkout GNU coreutils - uses: actions/checkout@v4 - with: - repository: 'coreutils/coreutils' - path: 'gnu' - ref: 'v9.4' - submodules: recursive - - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly - components: rustfmt - - uses: taiki-e/install-action@grcov - - uses: Swatinem/rust-cache@v2 - with: - workspaces: "./uutils -> target" - - name: Install dependencies - run: | - ## Install dependencies - sudo apt-get update - sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev - - name: Add various locales - run: | - ## Add various locales - echo "Before:" - locale -a - ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' - ## Some others need a French locale - sudo locale-gen - sudo locale-gen --keep-existing fr_FR - sudo locale-gen --keep-existing fr_FR.UTF-8 - sudo update-locale - echo "After:" - locale -a - - name: Build binaries - env: - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - RUSTDOCFLAGS: "-Cpanic=abort" - run: | - ## Build binaries - cd uutils - bash util/build-gnu.sh - - name: Run GNU tests - run: bash uutils/util/run-gnu-test.sh - - name: Generate coverage data (via `grcov`) - id: coverage - run: | - ## Generate coverage data - cd uutils - COVERAGE_REPORT_DIR="target/debug" - COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" - mkdir -p "${COVERAGE_REPORT_DIR}" - sudo chown -R "$(whoami)" "${COVERAGE_REPORT_DIR}" - # display coverage files - grcov . --output-type files --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique - # generate coverage report - grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" - echo "report=${COVERAGE_REPORT_FILE}" >> $GITHUB_OUTPUT - - name: Upload coverage results (to Codecov.io) - uses: codecov/codecov-action@v3 - with: - file: ${{ steps.coverage.outputs.report }} - flags: gnutests - name: gnutests - working-directory: uutils diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 86b5a89e83a..4f8edea3085 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -1,12 +1,14 @@ name: Android -# spell-checker:ignore TERMUX reactivecircus Swatinem noaudio pkill swiftshader dtolnay juliangruber +# spell-checker:ignore (people) reactivecircus Swatinem dtolnay juliangruber +# spell-checker:ignore (shell/tools) TERMUX nextest udevadm pkill +# spell-checker:ignore (misc) swiftshader playstore DATALOSS noaudio on: pull_request: push: branches: - - main + - '*' permissions: @@ -17,42 +19,115 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} +env: + TERMUX: v0.118.0 + KEY_POSTFIX: nextest+rustc-hash+adb+sshd+upgrade+XGB+inc18 + COMMON_EMULATOR_OPTIONS: -no-window -noaudio -no-boot-anim -camera-back none -gpu swiftshader_indirect -metrics-collection + EMULATOR_DISK_SIZE: 12GB + EMULATOR_HEAP_SIZE: 2048M + EMULATOR_BOOT_TIMEOUT: 1200 # 20min + jobs: test_android: name: Test builds - runs-on: macos-latest timeout-minutes: 90 strategy: fail-fast: false matrix: + os: [ubuntu-latest] # , macos-latest + cores: [4] # , 6 + ram: [4096, 8192] api-level: [28] - target: [default] - arch: [x86] # , arm64-v8a + target: [google_apis_playstore] + arch: [x86, x86_64] # , arm64-v8a + exclude: + - ram: 8192 + arch: x86 + - ram: 4096 + arch: x86_64 + runs-on: ${{ matrix.os }} env: - TERMUX: v0.118.0 + EMULATOR_RAM_SIZE: ${{ matrix.ram }} + EMULATOR_CORES: ${{ matrix.cores }} + RUNNER_OS: ${{ matrix.os }} + AVD_CACHE_KEY: "set later due to limitations of github actions not able to concatenate env variables" steps: + - name: Concatenate values to environment file + run: | + echo "AVD_CACHE_KEY=${{ matrix.os }}-${{ matrix.cores }}-${{ matrix.ram }}-${{ matrix.api-level }}-${{ matrix.target }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+${{ env.KEY_POSTFIX }}" >> $GITHUB_ENV + - name: Collect information about runner + if: always() + continue-on-error: true + run: | + hostname + uname -a + free -mh + df -Th + cat /proc/cpuinfo + - name: (Linux) create links from home to data partition + if: ${{ runner.os == 'Linux' }} + continue-on-error: true + run: | + ls -lah /mnt/ + cat /mnt/DATALOSS_WARNING_README.txt + sudo mkdir /mnt/data + sudo chmod a+rwx /mnt/data + mkdir /mnt/data/.android && ln -s /mnt/data/.android ~/.android + mkdir /mnt/data/work && ln -s /mnt/data/work ~/work + - name: Enable KVM group perms (linux hardware acceleration) + if: ${{ runner.os == 'Linux' }} + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Collect information about runner + if: always() + continue-on-error: true + run: | + free -mh + df -Th - name: Restore AVD cache uses: actions/cache/restore@v4 id: avd-cache + continue-on-error: true with: path: | ~/.android/avd/* ~/.android/avd/*/snapshots/* ~/.android/adb* ~/__rustc_hash__ - key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest+rustc-hash + key: avd-${{ env.AVD_CACHE_KEY }} + - name: Collect information about runner after AVD cache + if: always() + continue-on-error: true + run: | + free -mh + df -Th + ls -lah /mnt/data + du -sch /mnt/data + - name: Delete AVD Lockfile when run from cache + if: steps.avd-cache.outputs.cache-hit == 'true' + run: | + rm -f \ + ~/.android/avd/*.avd/*.lock \ + ~/.android/avd/*/*.lock - name: Create and cache emulator image if: steps.avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@v2 + uses: reactivecircus/android-emulator-runner@v2.34.0 with: api-level: ${{ matrix.api-level }} target: ${{ matrix.target }} arch: ${{ matrix.arch }} - ram-size: 2048M - disk-size: 7GB + ram-size: ${{ env.EMULATOR_RAM_SIZE }} + heap-size: ${{ env.EMULATOR_HEAP_SIZE }} + disk-size: ${{ env.EMULATOR_DISK_SIZE }} + cores: ${{ env.EMULATOR_CORES }} force-avd-creation: true - emulator-options: -no-window -no-snapshot-load -noaudio -no-boot-anim -camera-back none + emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-load + emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} script: | util/android-commands.sh init "${{ matrix.arch }}" "${{ matrix.api-level }}" "${{ env.TERMUX }}" - name: Save AVD cache @@ -64,12 +139,12 @@ jobs: ~/.android/avd/*/snapshots/* ~/.android/adb* ~/__rustc_hash__ - key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest+rustc-hash + key: avd-${{ env.AVD_CACHE_KEY }} - uses: juliangruber/read-file-action@v1 id: read_rustc_hash with: # ~ expansion didn't work - path: /Users/runner/__rustc_hash__ + path: ${{ runner.os == 'Linux' && '/home/runner/__rustc_hash__' || '/Users/runner/__rustc_hash__' }} trim: true - name: Restore rust cache id: rust-cache @@ -79,16 +154,25 @@ jobs: # The version vX at the end of the key is just a development version to avoid conflicts in # the github cache during the development of this workflow key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 + - name: Collect information about runner resources + if: always() + continue-on-error: true + run: | + free -mh + df -Th - name: Build and Test - uses: reactivecircus/android-emulator-runner@v2 + uses: reactivecircus/android-emulator-runner@v2.34.0 with: api-level: ${{ matrix.api-level }} target: ${{ matrix.target }} arch: ${{ matrix.arch }} - ram-size: 2048M - disk-size: 7GB + ram-size: ${{ env.EMULATOR_RAM_SIZE }} + heap-size: ${{ env.EMULATOR_HEAP_SIZE }} + disk-size: ${{ env.EMULATOR_DISK_SIZE }} + cores: ${{ env.EMULATOR_CORES }} force-avd-creation: false - emulator-options: -no-window -no-snapshot-save -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -snapshot ${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }} + emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-save -snapshot ${{ env.AVD_CACHE_KEY }} + emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} # This is not a usual script. Every line is executed in a separate shell with `sh -c`. If # one of the lines returns with error the whole script is failed (like running a script with # set -e) and in consequences the other lines (shells) are not executed. @@ -96,10 +180,28 @@ jobs: util/android-commands.sh sync_host util/android-commands.sh build util/android-commands.sh tests - if [[ "${{ steps.rust-cache.outputs.cache-hit }}" != 'true' ]]; then util/android-commands.sh sync_image; fi; exit 0 + if [ "${{ steps.rust-cache.outputs.cache-hit }}" != 'true' ]; then util/android-commands.sh sync_image; fi; exit 0 + - name: Collect information about runner resources + if: always() + continue-on-error: true + run: | + free -mh + df -Th - name: Save rust cache if: steps.rust-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: ~/__rust_cache__ key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 + - name: archive any output (error screenshots) + if: always() + uses: actions/upload-artifact@v4 + with: + name: test_output_${{ env.AVD_CACHE_KEY }} + path: output + - name: Collect information about runner resources + if: always() + continue-on-error: true + run: | + free -mh + df -Th diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 289830f8171..1c78fcaa2a3 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -1,12 +1,13 @@ name: Code Quality -# spell-checker:ignore TERMUX reactivecircus Swatinem noaudio pkill swiftshader dtolnay juliangruber +# spell-checker:ignore (people) reactivecircus Swatinem dtolnay juliangruber pell taplo +# spell-checker:ignore (misc) TERMUX noaudio pkill swiftshader esac sccache pcoreutils shopt subshell dequote on: pull_request: push: branches: - - main + - '*' env: # * style job configuration @@ -32,6 +33,8 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: stable @@ -44,16 +47,11 @@ jobs: ## VARs setup outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } # failure mode - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in + unset FAIL_ON_FAULT ; case "$STYLE_FAIL_ON_FAULT" in ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; esac; outputs FAIL_ON_FAULT FAULT_TYPE - # target-specific options - # * CARGO_FEATURES_OPTION - CARGO_FEATURES_OPTION='' ; - if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features "${{ matrix.job.features }}"' ; fi - outputs CARGO_FEATURES_OPTION - name: "`cargo fmt` testing" shell: bash run: | @@ -80,13 +78,15 @@ jobs: - { os: windows-latest , features: feat_os_windows } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: stable components: clippy - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Initialize workflow variables id: vars shell: bash @@ -94,39 +94,26 @@ jobs: ## VARs setup outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } # failure mode - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in + unset FAIL_ON_FAULT ; case "$STYLE_FAIL_ON_FAULT" in ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; esac; outputs FAIL_ON_FAULT FAULT_TYPE - # target-specific options - # * CARGO_FEATURES_OPTION - CARGO_FEATURES_OPTION='--all-features' ; - if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features ${{ matrix.job.features }}' ; fi - outputs CARGO_FEATURES_OPTION - # * determine sub-crate utility list - UTILITY_LIST="$(./util/show-utils.sh ${CARGO_FEATURES_OPTION})" - echo UTILITY_LIST=${UTILITY_LIST} - CARGO_UTILITY_LIST_OPTIONS="$(for u in ${UTILITY_LIST}; do echo -n "-puu_${u} "; done;)" - outputs CARGO_UTILITY_LIST_OPTIONS - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites - case '${{ matrix.job.os }}' in - macos-latest) brew install coreutils ;; # needed for show-utils.sh - esac - name: "`cargo clippy` lint testing" - shell: bash - run: | - ## `cargo clippy` lint testing - unset fault - CLIPPY_FLAGS="-W clippy::default_trait_access -W clippy::manual_string_new -W clippy::cognitive_complexity -W clippy::implicit_clone" - fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" - fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') - # * convert any warnings to GHA UI annotations; ref: - S=$(cargo clippy --all-targets ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} -- ${CLIPPY_FLAGS} -D warnings 2>&1) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*$/::${fault_type} file=\2,line=\3,col=\4::${fault_prefix}: \`cargo clippy\`: \1 (file:'\2', line:\3)/p;" -e '}' ; fault=true ; } - if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + retry_on: error + timeout_minutes: 90 + shell: bash + command: | + ## `cargo clippy` lint testing + unset fault + fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" + fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') + # * convert any warnings to GHA UI annotations; ref: + S=$(cargo clippy --all-targets --features ${{ matrix.job.features }} --tests -pcoreutils -- -D warnings 2>&1) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*$/::${fault_type} file=\2,line=\3,col=\4::${fault_prefix}: \`cargo clippy\`: \1 (file:'\2', line:\3)/p;" -e '}' ; fault=true ; } + if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi style_spellcheck: name: Style/spelling @@ -137,6 +124,8 @@ jobs: - { os: ubuntu-latest , features: feat_os_unix } steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Initialize workflow variables id: vars shell: bash @@ -144,7 +133,7 @@ jobs: ## VARs setup outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } # failure mode - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in + unset FAIL_ON_FAULT ; case "$STYLE_FAIL_ON_FAULT" in ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; esac; @@ -152,10 +141,7 @@ jobs: - name: Install/setup prerequisites shell: bash run: | - ## Install/setup prerequisites - # * pin installed cspell to v4.2.8 (cspell v5+ is broken for NodeJS < v12) - ## maint: [2021-11-10; rivy] `cspell` version may be advanced to v5 when used with NodeJS >= v12 - sudo apt-get -y update ; sudo apt-get -y install npm ; sudo npm install cspell@4.2.8 -g ; + sudo apt-get -y update ; sudo apt-get -y install npm ; sudo npm install cspell -g ; - name: Run `cspell` shell: bash run: | @@ -167,10 +153,7 @@ jobs: cfg_files=($(shopt -s nullglob ; echo {.vscode,.}/{,.}c[sS]pell{.json,.config{.js,.cjs,.json,.yaml,.yml},.yaml,.yml} ;)) cfg_file=${cfg_files[0]} unset CSPELL_CFG_OPTION ; if [ -n "$cfg_file" ]; then CSPELL_CFG_OPTION="--config $cfg_file" ; fi - # * `cspell` - ## maint: [2021-11-10; rivy] the `--no-progress` option for `cspell` is a `cspell` v5+ option - # S=$(cspell ${CSPELL_CFG_OPTION} --no-summary --no-progress "**/*") && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n "s/${PWD//\//\\/}\/(.*):(.*):(.*) - (.*)/::${fault_type} file=\1,line=\2,col=\3::${fault_type^^}: \4 (file:'\1', line:\2)/p" ; fault=true ; true ; } - S=$(cspell ${CSPELL_CFG_OPTION} --no-summary "**/*") && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n "s/${PWD//\//\\/}\/(.*):(.*):(.*) - (.*)/::${fault_type} file=\1,line=\2,col=\3::${fault_type^^}: \4 (file:'\1', line:\2)/p" ; fault=true ; true ; } + S=$(cspell ${CSPELL_CFG_OPTION} --no-summary --no-progress .) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n "s/${PWD//\//\\/}\/(.*):(.*):(.*) - (.*)/::${fault_type} file=\1,line=\2,col=\3::${fault_type^^}: \4 (file:'\1', line:\2)/p" ; fault=true ; true ; } if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi toml_format: @@ -179,6 +162,32 @@ jobs: steps: - name: Clone repository uses: actions/checkout@v4 + with: + persist-credentials: false - name: Check run: npx --yes @taplo/cli fmt --check + + python: + name: Style/Python + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: ruff + uses: astral-sh/ruff-action@v3 + with: + src: "./util" + + - name: ruff - format + uses: astral-sh/ruff-action@v3 + with: + src: "./util" + args: format --check + - name: Run Python unit tests + shell: bash + run: | + python3 -m unittest util/test_compare_test_results.py diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index 25655f0917f..6e96bfd9599 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -1,6 +1,6 @@ name: FreeBSD -# spell-checker:ignore sshfs usesh vmactions taiki Swatinem esac fdescfs fdesc +# spell-checker:ignore sshfs usesh vmactions taiki Swatinem esac fdescfs fdesc sccache nextest copyback env: # * style job configuration @@ -10,7 +10,7 @@ on: pull_request: push: branches: - - main + - '*' permissions: contents: read # to fetch code (actions/checkout) @@ -29,17 +29,19 @@ jobs: fail-fast: false matrix: job: - - { os: ubuntu-22.04 , features: unix } + - { os: ubuntu-24.04 , features: unix } env: SCCACHE_GHA_ENABLED: "true" RUSTC_WRAPPER: "sccache" steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Prepare, build and test - uses: vmactions/freebsd-vm@v1.0.5 + uses: vmactions/freebsd-vm@v1.2.0 with: usesh: true sync: rsync @@ -105,7 +107,7 @@ jobs: if [ -z "\${FAULT}" ]; then echo "## cargo clippy lint testing" # * convert any warnings to GHA UI annotations; ref: - S=\$(cargo clippy --all-targets \${CARGO_UTILITY_LIST_OPTIONS} -- -W clippy::manual_string_new -D warnings 2>&1) && printf "%s\n" "\$S" || { printf "%s\n" "\$S" ; printf "%s" "\$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*\$/::\${FAULT_TYPE} file=\2,line=\3,col=\4::\${FAULT_PREFIX}: \\\`cargo clippy\\\`: \1 (file:'\2', line:\3)/p;" -e '}' ; FAULT=true ; } + S=\$(cargo clippy --all-targets \${CARGO_UTILITY_LIST_OPTIONS} -- -D warnings 2>&1) && printf "%s\n" "\$S" || { printf "%s\n" "\$S" ; printf "%s" "\$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*\$/::\${FAULT_TYPE} file=\2,line=\3,col=\4::\${FAULT_PREFIX}: \\\`cargo clippy\\\`: \1 (file:'\2', line:\3)/p;" -e '}' ; FAULT=true ; } fi # Clean to avoid to rsync back the files cargo clean @@ -120,18 +122,20 @@ jobs: fail-fast: false matrix: job: - - { os: ubuntu-22.04 , features: unix } + - { os: ubuntu-24.04 , features: unix } env: mem: 4096 SCCACHE_GHA_ENABLED: "true" RUSTC_WRAPPER: "sccache" steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Prepare, build and test - uses: vmactions/freebsd-vm@v1.0.5 + uses: vmactions/freebsd-vm@v1.2.0 with: usesh: true sync: rsync diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index c60c01ff4be..e7da4b5926d 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -1,12 +1,12 @@ name: Fuzzing -# spell-checker:ignore fuzzer +# spell-checker:ignore fuzzer dtolnay Swatinem on: pull_request: push: branches: - - main + - '*' permissions: contents: read # to fetch code (actions/checkout) @@ -22,6 +22,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@nightly - name: Install `cargo-fuzz` run: cargo install cargo-fuzz @@ -34,7 +36,7 @@ jobs: fuzz-run: needs: fuzz-build - name: Run the fuzzers + name: Fuzz runs-on: ubuntu-latest timeout-minutes: 5 env: @@ -46,18 +48,25 @@ jobs: # https://github.com/uutils/coreutils/issues/5311 - { name: fuzz_date, should_pass: false } - { name: fuzz_expr, should_pass: true } - - { name: fuzz_printf, should_pass: false } + - { name: fuzz_printf, should_pass: true } - { name: fuzz_echo, should_pass: true } - { name: fuzz_seq, should_pass: false } - { name: fuzz_sort, should_pass: false } - { name: fuzz_wc, should_pass: false } - { name: fuzz_cut, should_pass: false } - { name: fuzz_split, should_pass: false } + - { name: fuzz_tr, should_pass: false } + - { name: fuzz_env, should_pass: false } + - { name: fuzz_cksum, should_pass: false } - { name: fuzz_parse_glob, should_pass: true } - { name: fuzz_parse_size, should_pass: true } - { name: fuzz_parse_time, should_pass: true } + - { name: fuzz_seq_parse_number, should_pass: true } + steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@nightly - name: Install `cargo-fuzz` run: cargo install cargo-fuzz @@ -72,13 +81,201 @@ jobs: path: | fuzz/corpus/${{ matrix.test-target.name }} - name: Run ${{ matrix.test-target.name }} for XX seconds + id: run_fuzzer shell: bash continue-on-error: ${{ !matrix.test-target.name.should_pass }} run: | - cargo +nightly fuzz run ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -detect_leaks=0 + mkdir -p fuzz/stats + STATS_FILE="fuzz/stats/${{ matrix.test-target.name }}.txt" + cargo +nightly fuzz run ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE" + + # Extract key stats from the output + if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then + RUNS=$(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') + echo "runs=$RUNS" >> "$GITHUB_OUTPUT" + else + echo "runs=unknown" >> "$GITHUB_OUTPUT" + fi + + if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then + EXEC_RATE=$(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') + echo "exec_rate=$EXEC_RATE" >> "$GITHUB_OUTPUT" + else + echo "exec_rate=unknown" >> "$GITHUB_OUTPUT" + fi + + if grep -q "stat::new_units_added" "$STATS_FILE"; then + NEW_UNITS=$(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') + echo "new_units=$NEW_UNITS" >> "$GITHUB_OUTPUT" + else + echo "new_units=unknown" >> "$GITHUB_OUTPUT" + fi + + # Save should_pass value to file for summary job to use + echo "${{ matrix.test-target.should_pass }}" > "fuzz/stats/${{ matrix.test-target.name }}.should_pass" + + # Print stats to job output for immediate visibility + echo "----------------------------------------" + echo "FUZZING STATISTICS FOR ${{ matrix.test-target.name }}" + echo "----------------------------------------" + echo "Runs: $(grep -q "stat::number_of_executed_units" "$STATS_FILE" && grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}' || echo "unknown")" + echo "Execution Rate: $(grep -q "stat::average_exec_per_sec" "$STATS_FILE" && grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}' || echo "unknown") execs/sec" + echo "New Units: $(grep -q "stat::new_units_added" "$STATS_FILE" && grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}' || echo "unknown")" + echo "Expected: ${{ matrix.test-target.name.should_pass }}" + if grep -q "SUMMARY: " "$STATS_FILE"; then + echo "Status: $(grep "SUMMARY: " "$STATS_FILE" | head -1)" + else + echo "Status: Completed" + fi + echo "----------------------------------------" + + # Add summary to GitHub step summary + echo "### Fuzzing Results for ${{ matrix.test-target.name }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY + echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY + + if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then + echo "| Runs | $(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY + fi + + if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then + echo "| Execution Rate | $(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') execs/sec |" >> $GITHUB_STEP_SUMMARY + fi + + if grep -q "stat::new_units_added" "$STATS_FILE"; then + echo "| New Units | $(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY + fi + + echo "| Should pass | ${{ matrix.test-target.should_pass }} |" >> $GITHUB_STEP_SUMMARY + + if grep -q "SUMMARY: " "$STATS_FILE"; then + echo "| Status | $(grep "SUMMARY: " "$STATS_FILE" | head -1) |" >> $GITHUB_STEP_SUMMARY + else + echo "| Status | Completed |" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY - name: Save Corpus Cache uses: actions/cache/save@v4 with: key: corpus-cache-${{ matrix.test-target.name }} path: | fuzz/corpus/${{ matrix.test-target.name }} + - name: Upload Stats + uses: actions/upload-artifact@v4 + with: + name: fuzz-stats-${{ matrix.test-target.name }} + path: | + fuzz/stats/${{ matrix.test-target.name }}.txt + fuzz/stats/${{ matrix.test-target.name }}.should_pass + retention-days: 5 + fuzz-summary: + needs: fuzz-run + name: Fuzzing Summary + runs-on: ubuntu-latest + if: always() + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Download all stats + uses: actions/download-artifact@v4 + with: + path: fuzz/stats-artifacts + pattern: fuzz-stats-* + merge-multiple: true + - name: Prepare stats directory + run: | + mkdir -p fuzz/stats + # Debug: List content of stats-artifacts directory + echo "Contents of stats-artifacts directory:" + find fuzz/stats-artifacts -type f | sort + + # Extract files from the artifact directories - handle nested directories + find fuzz/stats-artifacts -type f -name "*.txt" -exec cp {} fuzz/stats/ \; + find fuzz/stats-artifacts -type f -name "*.should_pass" -exec cp {} fuzz/stats/ \; + + # Debug information + echo "Contents of stats directory after extraction:" + ls -la fuzz/stats/ + echo "Contents of should_pass files (if any):" + cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found" + - name: Generate Summary + run: | + echo "# Fuzzing Summary" > fuzzing_summary.md + echo "" >> fuzzing_summary.md + echo "| Target | Runs | Exec/sec | New Units | Should pass | Status |" >> fuzzing_summary.md + echo "|--------|------|----------|-----------|-------------|--------|" >> fuzzing_summary.md + + TOTAL_RUNS=0 + TOTAL_NEW_UNITS=0 + + for stat_file in fuzz/stats/*.txt; do + TARGET=$(basename "$stat_file" .txt) + SHOULD_PASS_FILE="${stat_file%.*}.should_pass" + + # Get expected status + if [ -f "$SHOULD_PASS_FILE" ]; then + EXPECTED=$(cat "$SHOULD_PASS_FILE") + else + EXPECTED="unknown" + fi + + # Extract runs + if grep -q "stat::number_of_executed_units" "$stat_file"; then + RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}') + TOTAL_RUNS=$((TOTAL_RUNS + RUNS)) + else + RUNS="unknown" + fi + + # Extract execution rate + if grep -q "stat::average_exec_per_sec" "$stat_file"; then + EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}') + else + EXEC_RATE="unknown" + fi + + # Extract new units added + if grep -q "stat::new_units_added" "$stat_file"; then + NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}') + if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then + TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS)) + fi + else + NEW_UNITS="unknown" + fi + + # Extract status + if grep -q "SUMMARY: " "$stat_file"; then + STATUS=$(grep "SUMMARY: " "$stat_file" | head -1) + else + STATUS="Completed" + fi + + echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md + done + + echo "" >> fuzzing_summary.md + echo "## Overall Statistics" >> fuzzing_summary.md + echo "" >> fuzzing_summary.md + echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md + echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md + echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md + + # Add count by expected status + echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md + echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md + + # Write to GitHub step summary + cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY + - name: Show Summary + run: | + cat fuzzing_summary.md + - name: Upload Summary + uses: actions/upload-artifact@v4 + with: + name: fuzzing-summary + path: fuzzing_summary.md + retention-days: 5 diff --git a/.github/workflows/ignore-intermittent.txt b/.github/workflows/ignore-intermittent.txt index e163202a2ca..9e0e2ab0df6 100644 --- a/.github/workflows/ignore-intermittent.txt +++ b/.github/workflows/ignore-intermittent.txt @@ -1,3 +1,6 @@ tests/tail/inotify-dir-recreate -tests/misc/timeout +tests/timeout/timeout tests/rm/rm1 +tests/misc/stdbuf +tests/misc/usage_vs_getopt +tests/misc/tee diff --git a/.gitignore b/.gitignore index ed4e54ec5bc..7528e5f5380 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +# spell-checker:ignore (misc) direnv + target/ +coverage/ /src/*/gen_table /build/ /tmp/ @@ -10,9 +13,11 @@ target/ .*.swp .*.swo .idea -Cargo.lock lib*.a /docs/_build *.iml ### macOS ### .DS_Store + +### direnv ### +/.direnv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f90466bed2f..53e879d09a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,14 +4,20 @@ repos: - id: rust-linting name: Rust linting description: Run cargo fmt on files included in the commit. - entry: cargo +nightly fmt -- + entry: cargo +stable fmt -- pass_filenames: true types: [file, rust] language: system - id: rust-clippy name: Rust clippy description: Run cargo clippy on files included in the commit. - entry: cargo +nightly clippy --workspace --all-targets --all-features -- + entry: cargo +stable clippy --workspace --all-targets --all-features -- pass_filenames: false types: [file, rust] language: system + - id: cspell + name: Code spell checker (cspell) + description: Run cspell to check for spelling errors. + entry: cspell -- + pass_filenames: true + language: system diff --git a/.vscode/cSpell.json b/.vscode/cSpell.json index 6ceb038c218..01e192d59ba 100644 --- a/.vscode/cSpell.json +++ b/.vscode/cSpell.json @@ -1,4 +1,5 @@ // `cspell` settings +// spell-checker:ignore oranda { // version of the setting file "version": "0.2", @@ -18,6 +19,7 @@ // files to ignore (globs supported) "ignorePaths": [ + ".git/**", "Cargo.lock", "oranda.json", "target/**", @@ -27,6 +29,8 @@ "**/*.svg" ], + "enableGlobDot": true, + // words to ignore (even if they are in the flagWords) "ignoreWords": [], diff --git a/.vscode/cspell.dictionaries/acronyms+names.wordlist.txt b/.vscode/cspell.dictionaries/acronyms+names.wordlist.txt index 4a59ed094bd..8993f5d6a31 100644 --- a/.vscode/cspell.dictionaries/acronyms+names.wordlist.txt +++ b/.vscode/cspell.dictionaries/acronyms+names.wordlist.txt @@ -46,6 +46,7 @@ Codacy Cygwin Deno EditorConfig +EPEL FreeBSD Gmail GNU diff --git a/.vscode/cspell.dictionaries/jargon.wordlist.txt b/.vscode/cspell.dictionaries/jargon.wordlist.txt index 20e26990f3b..6358f3c7682 100644 --- a/.vscode/cspell.dictionaries/jargon.wordlist.txt +++ b/.vscode/cspell.dictionaries/jargon.wordlist.txt @@ -10,8 +10,10 @@ bytewise canonicalization canonicalize canonicalizing +capget codepoint codepoints +codeready codegen colorizable colorize @@ -45,6 +47,8 @@ flamegraph fsxattr fullblock getfacl +getfattr +getopt gibi gibibytes glob @@ -64,6 +68,7 @@ kibi kibibytes libacl lcase +llistxattr lossily lstat mebi @@ -74,6 +79,7 @@ microbenchmarks microbenchmarking multibyte multicall +nmerge noatime nocache nocreat @@ -84,6 +90,7 @@ nolinks nonblock nonportable nonprinting +nonseekable notrunc noxfer ofile @@ -106,7 +113,9 @@ seedable semver semiprime semiprimes +setcap setfacl +setfattr shortcode shortcodes siginfo @@ -135,11 +144,14 @@ whitespace wordlist wordlists xattrs +xpass # * abbreviations consts deps dev +fdlimit +inacc maint proc procs @@ -155,3 +167,9 @@ retval subdir val vals +inval +nofield + +# * clippy +uninlined +nonminimal diff --git a/.vscode/cspell.dictionaries/shell.wordlist.txt b/.vscode/cspell.dictionaries/shell.wordlist.txt index 95dea94a7cd..b1ddec7a4e0 100644 --- a/.vscode/cspell.dictionaries/shell.wordlist.txt +++ b/.vscode/cspell.dictionaries/shell.wordlist.txt @@ -25,6 +25,7 @@ sudoedit tcsh tzselect urandom +VARNAME wtmp zsh @@ -102,3 +103,4 @@ xargs # * directories sbin +libexec diff --git a/.vscode/cspell.dictionaries/workspace.wordlist.txt b/.vscode/cspell.dictionaries/workspace.wordlist.txt index c3c854a4cd5..3757980d334 100644 --- a/.vscode/cspell.dictionaries/workspace.wordlist.txt +++ b/.vscode/cspell.dictionaries/workspace.wordlist.txt @@ -20,7 +20,6 @@ exacl filetime formatteriteminfo fsext -fundu getopts getrandom globset @@ -136,6 +135,7 @@ vmsplice # * vars/libc COMFOLLOW +EXDEV FILENO FTSENT HOSTSIZE @@ -166,6 +166,7 @@ RTLD_NEXT RTLD SIGINT SIGKILL +SIGSTOP SIGTERM SYS_fdatasync SYS_syncfs @@ -323,6 +324,7 @@ libc libstdbuf musl tmpd +uchild ucmd ucommand utmpx @@ -331,7 +333,11 @@ uucore_procs uudoc uumain uutil +uutests uutils # * function names getcwd + +# * other +algs diff --git a/.vscode/extensions.json b/.vscode/extensions.json index b9c161a4c28..7ee2695db0a 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,4 +1,4 @@ -// spell-checker:ignore (misc) matklad +// spell-checker:ignore (misc) matklad foxundermoon // see for the documentation about the extensions.json format // * // "foxundermoon.shell-format" ~ shell script formatting ; note: ENABLE "Use EditorConfig" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b10d3d11472..2e609083fc8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,9 @@ Now follows a very important warning: > uutils is original code and cannot contain any code from GNU or > other implementations. This means that **we cannot accept any changes based on > the GNU source code**. To make sure that cannot happen, **you cannot link to -> the GNU source code** either. +> the GNU source code** either. It is however possible to look at other implementations +> under a BSD or MIT license like [Apple's implementation](https://github.com/apple-oss-distributions/file_cmds/) +> or [OpenBSD](https://github.com/openbsd/src/tree/master/bin). Finally, feel free to join our [Discord](https://discord.gg/wQVJbvJ)! @@ -27,13 +29,15 @@ Finally, feel free to join our [Discord](https://discord.gg/wQVJbvJ)! uutils is a big project consisting of many parts. Here are the most important parts for getting started: -- [`src/uu`](./src/uu/): The code for all utilities -- [`src/uucore`](./src/uucore/): Crate containing all the shared code between +- [`src/uu`](https://github.com/uutils/coreutils/tree/main/src/uu/): The code for all utilities +- [`src/uucore`](https://github.com/uutils/coreutils/tree/main/src/uucore/): Crate containing all the shared code between the utilities. -- [`tests/by-util`](./tests/by-util/): The tests for all utilities. -- [`src/bin/coreutils.rs`](./src/bin/coreutils.rs): Code for the multicall +- [`tests/by-util`](https://github.com/uutils/coreutils/tree/main/tests/by-util/): The tests for all utilities. +- [`src/bin/coreutils.rs`](https://github.com/uutils/coreutils/tree/main/src/bin/coreutils.rs): Code for the multicall binary. -- [`docs`](./docs/src): the documentation for the website +- [`docs`](https://github.com/uutils/coreutils/tree/main/docs/src): the documentation for the website +- [`tests/uutests/`](https://github.com/uutils/coreutils/tree/main/tests/uutests/): + Crate implementing the various functions to test uutils commands. Each utility is defined as a separate crate. The structure of each of these crates is as follows: @@ -202,8 +206,8 @@ To ensure easy collaboration, we have guidelines for using Git and GitHub. ### Commit messages -You can read this section in the Git book to learn how to write good commit -messages: https://git-scm.com/book/ch5-2.html. +You can read [this section in the Git book](https://git-scm.com/book/ms/v2/Distributed-Git-Contributing-to-a-Project) to learn how to write good commit +messages. In addition, here are a few examples for a summary line when committing to uutils: @@ -255,8 +259,7 @@ CI. However, you can use `#[cfg(...)]` attributes to create platform dependent features. **Tip:** For Windows, Microsoft provides some images (VMWare, Hyper-V, -VirtualBox and Parallels) for development: - +VirtualBox and Parallels) for development [here](https://developer.microsoft.com/windows/downloads/virtual-machines/). ## Improving the GNU compatibility @@ -303,6 +306,7 @@ completions: - [OpenBSD](https://github.com/openbsd/src/tree/master/bin) - [Busybox](https://github.com/mirror/busybox/tree/master/coreutils) - [Toybox (Android)](https://github.com/landley/toybox/tree/master/toys/posix) +- [Mac OS](https://github.com/apple-oss-distributions/file_cmds/) - [V lang](https://github.com/vlang/coreutils) - [SerenityOS](https://github.com/SerenityOS/serenity/tree/master/Userland/Utilities) - [Initial Unix](https://github.com/dspinellis/unix-history-repo) diff --git a/Cargo.lock b/Cargo.lock index e8c0ce0ca28..70657f90ee3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,22 +1,28 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -32,78 +38,99 @@ dependencies = [ "libc", ] +[[package]] +name = "ansi-width" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219e3ce6f2611d83b51ec2098a12702112c29e57203a6b0a0929b2cddb486608" +dependencies = [ + "unicode-width 0.1.14", +] + [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "once_cell", + "windows-sys 0.59.0", +] + +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", ] [[package]] name = "arrayref" -version = "0.3.6" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bigdecimal" -version = "0.4.0" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5274a6b6e0ee020148397245b973e30163b7bffbc6d473613f850cb99888581e" +checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" dependencies = [ + "autocfg", "libm", "num-bigint", "num-integer", @@ -119,26 +146,44 @@ dependencies = [ "compare", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" -version = "0.63.0" +version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d860121800b2a9a94f9b5604b332d5cffb234ce17609ea479d723dbc9d3885" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools 0.13.0", "log", - "peeking_take_while", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 1.0.109", - "which", + "syn", ] [[package]] @@ -149,15 +194,27 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] [[package]] name = "blake2b_simd" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23285ad32269793932e830392f2fe2f83e26488fd3ec778883a93c8323735780" +checksum = "06e903a20b159e944f91ec8499fe1e55651480c541ea0a584f5d967c49ad9d99" dependencies = [ "arrayref", "arrayvec", @@ -166,9 +223,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" dependencies = [ "arrayref", "arrayvec", @@ -179,18 +236,18 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] name = "bstr" -version = "1.9.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", "regex-automata", @@ -199,15 +256,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytecount" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" [[package]] name = "byteorder" @@ -217,9 +274,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cc" -version = "1.0.79" +version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +dependencies = [ + "shlex", +] [[package]] name = "cexpr" @@ -227,7 +287,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -236,23 +296,50 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" -version = "0.4.32" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41daef31d7a747c5c847246f36de49ced6f7403b4cdabc807a97b5cc184cda7a" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-targets 0.52.0", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efdce149c370f133a071ca8ef6ea340b7b88748ab0810097a9e2976eaa34b4f3" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" +dependencies = [ + "parse-zoneinfo", + "phf_codegen", ] [[package]] name = "clang-sys" -version = "1.4.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -261,46 +348,46 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.2" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" dependencies = [ "anstream", "anstyle", "clap_lex", "strsim", - "terminal_size 0.2.6", + "terminal_size", ] [[package]] name = "clap_complete" -version = "4.4.0" +version = "4.5.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "586a385f7ef2f8b4d86bddaa0c094794e7ccbfe5ffef1f434fe928143fc783a5" +checksum = "c91d3baa3bcd889d60e6ef28874126a0b384fd225ab83aa6d8a801c519194ce1" dependencies = [ "clap", ] [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "clap_mangen" -version = "0.2.9" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0f09a0ca8f0dd8ac92c546b426f466ef19828185c6d504c80c48c9c2768ed9" +checksum = "724842fa9b144f9b89b3f3d371a89f3455eea660361d13a554f68f8ae5d6c13a" dependencies = [ "clap", "roff", @@ -308,9 +395,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "compare" @@ -320,22 +407,22 @@ checksum = "120133d4db2ec47efe2e26502ee984747630c67f51974fca0b6c1340cf2368d3" [[package]] name = "console" -version = "0.15.8" +version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ "encode_unicode", - "lazy_static", "libc", - "unicode-width", - "windows-sys 0.52.0", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", ] [[package]] name = "const-random" -version = "0.1.16" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11df32a13d7892ec42d51d3d175faba5211ffe13ed25d4fb348ac9e9ce835593" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", ] @@ -346,57 +433,59 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "tiny-keccak", ] [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] -name = "conv" -version = "0.3.3" +name = "convert_case" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ff10625fd0ac447827aa30ea8b861fead473bb60aeb73af6c1c58caf0d1299" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" dependencies = [ - "custom_derive", + "unicode-segmentation", ] [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "coreutils" -version = "0.0.24" +version = "0.0.30" dependencies = [ + "bincode", "chrono", "clap", "clap_complete", "clap_mangen", - "conv", + "ctor", "filetime", "glob", "hex-literal", "libc", "nix", - "once_cell", + "num-prime", "phf", "phf_codegen", "pretty_assertions", "procfs", - "rand", - "rand_pcg", + "rand 0.9.1", "regex", "rlimit", "rstest", "selinux", + "serde", + "serde-big-array", "sha1", "tempfile", "textwrap", @@ -506,6 +595,7 @@ dependencies = [ "uu_yes", "uucore", "uuhelp_parser", + "uutests", "walkdir", "xattr", "zip", @@ -523,44 +613,44 @@ dependencies = [ [[package]] name = "cpp" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa65869ef853e45c60e9828aa08cdd1398cb6e13f3911d9cb2a079b144fcd64" +checksum = "f36bcac3d8234c1fb813358e83d1bb6b0290a3d2b3b5efc6b88bfeaf9d8eec17" dependencies = [ "cpp_macros", ] [[package]] name = "cpp_build" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e361fae2caf9758164b24da3eedd7f7d7451be30d90d8e7b5d2be29a2f0cf5b" +checksum = "27f8638c97fbd79cc6fc80b616e0e74b49bac21014faed590bbc89b7e2676c90" dependencies = [ "cc", "cpp_common", "lazy_static", "proc-macro2", "regex", - "syn 2.0.23", + "syn", "unicode-xid", ] [[package]] name = "cpp_common" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1a2532e4ed4ea13031c13bc7bc0dbca4aae32df48e9d77f0d1e743179f2ea1" +checksum = "25fcfea2ee05889597d35e986c2ad0169694320ae5cc8f6d2640a4bb8a884560" dependencies = [ "lazy_static", "proc-macro2", - "syn 2.0.23", + "syn", ] [[package]] name = "cpp_macros" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47ec9cc90633446f779ef481a9ce5a0077107dd5b87016440448d908625a83fd" +checksum = "d156158fe86e274820f5a53bc9edb0885a6e7113909497aa8d883b69dd171871" dependencies = [ "aho-corasick", "byteorder", @@ -568,79 +658,66 @@ dependencies = [ "lazy_static", "proc-macro2", "quote", - "syn 2.0.23", + "syn", ] [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.17" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if", -] +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" -version = "0.27.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.9.0", "crossterm_winapi", - "libc", + "derive_more", + "document-features", + "filedescriptor", "mio", "parking_lot", + "rustix 1.0.1", "signal-hook", "signal-hook-mio", "winapi", @@ -657,9 +734,9 @@ dependencies = [ [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-common" @@ -672,32 +749,42 @@ dependencies = [ ] [[package]] -name = "ctrlc" -version = "3.4.1" +name = "ctor" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e95fbd621905b854affdc67943b043a0fbb6ed7385fd5a25650d19a8a6cfdf" +checksum = "a4735f265ba6a1188052ca32d461028a7d1125868be18e287e756019da7607b5" dependencies = [ - "nix", - "windows-sys 0.48.0", + "ctor-proc-macro", + "dtor", ] [[package]] -name = "custom_derive" -version = "0.1.7" +name = "ctor-proc-macro" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f211af61d8efdd104f96e57adf5e426ba1bc3ed7a4ead616e15e5881fd79c4d" + +[[package]] +name = "ctrlc" +version = "3.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" +checksum = "697b5419f348fd5ae2478e8018cb016c00a5881c7f46c717de98ffd135a5651c" +dependencies = [ + "nix", + "windows-sys 0.59.0", +] [[package]] name = "data-encoding" -version = "2.5.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "data-encoding-macro" -version = "0.1.14" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20c01c06f5f429efdf2bae21eb67c28b3df3cf85b7dd2d8ef09c0838dac5d33e" +checksum = "47ce6c96ea0102f01122a185683611bd5ac8d99e62bc59dd12e6bda344ee673d" dependencies = [ "data-encoding", "data-encoding-macro-internal", @@ -705,12 +792,53 @@ dependencies = [ [[package]] name = "data-encoding-macro-internal" -version = "0.1.12" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0047d07f2c89b17dd631c80450d69841a6b5d7fb17278cbc43d7e4cfcf2576f3" +checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn", +] + +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -731,9 +859,9 @@ dependencies = [ [[package]] name = "dlv-list" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d529fd73d344663edfd598ccb3f344e46034db51ebd103518eae34338248ad73" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" dependencies = [ "const-random", ] @@ -750,51 +878,71 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "document-features" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +dependencies = [ + "litrs", +] + +[[package]] +name = "dtor" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cbdf2ad6846025e8e25df05171abfb30e3ababa12ee0a0e44b9bbe570633a8" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7454e41ff9012c00d53cf7f475c5e3afa3b91b7c90568495495e8d9bf47a1055" + [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" -version = "1.8.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] -name = "env_logger" -version = "0.8.4" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" -dependencies = [ - "log", - "regex", -] +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "exacl" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c695152c1c2777163ea93fff517edc6dd1f8fc226c14b0d60cdcde0beb316d9f" +checksum = "22be12de19decddab85d09f251ec8363f060ccb22ec9c81bc157c0c8433946d8" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.9.0", "log", "scopeguard", "uuid", @@ -802,9 +950,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file_diff" @@ -812,23 +960,34 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31a7a908b8f32538a2143e59a6e4e2508988832d5d4d6f7c156b3cbc762643a5" +[[package]] +name = "filedescriptor" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" +dependencies = [ + "libc", + "thiserror 1.0.69", + "winapi", +] + [[package]] name = "filetime" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", - "redox_syscall", - "windows-sys 0.52.0", + "libredox", + "windows-sys 0.59.0", ] [[package]] name = "flate2" -version = "1.0.24" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", "miniz_oxide", @@ -840,6 +999,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -857,119 +1022,58 @@ dependencies = [ [[package]] name = "fts-sys" -version = "0.2.4" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a66c0a21e344f20c87b4ca12643cf4f40a7018f132c98d344e989b959f49dd1" +checksum = "43119ec0f2227f8505c8bb6c60606b5eefc328607bfe1a421e561c4decfa02ab" dependencies = [ "bindgen", "libc", ] [[package]] -name = "fundu" +name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c04cb831a8dccadfe3774b07cba4574a1ec24974d761510e65d8a543c2d7cb4" -dependencies = [ - "fundu-core", -] - -[[package]] -name = "fundu-core" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a889e633afd839fb5b04fe53adfd588cefe518e71ec8d3c929698c6daf2acd" - -[[package]] -name = "futures" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" -dependencies = [ - "futures-core", - "futures-sink", -] +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures-core" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" - -[[package]] -name = "futures-executor" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn", ] -[[package]] -name = "futures-sink" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" - [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "futures-channel", "futures-core", - "futures-io", "futures-macro", - "futures-sink", "futures-task", - "memchr", "pin-project-lite", "pin-utils", "slab", @@ -983,9 +1087,9 @@ checksum = "1d758ba1b47b00caf47f24925c0074ecb20d6dfcffe7f6d53395c0465674841a" [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -993,26 +1097,38 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] -name = "glob" +name = "getrandom" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "half" -version = "2.3.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", @@ -1020,15 +1136,20 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.13.2" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] -name = "hermit-abi" -version = "0.3.2" +name = "hashbrown" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "hex" @@ -1038,33 +1159,34 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hex-literal" -version = "0.4.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" +checksum = "bcaaec4551594c969335c98c903c1397853d4198408ea609190f420500f6be71" [[package]] name = "hostname" -version = "0.3.1" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" dependencies = [ + "cfg-if", "libc", - "match_cfg", - "winapi", + "windows-link", ] [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "winapi", + "windows-core", ] [[package]] @@ -1076,25 +1198,36 @@ dependencies = [ "cc", ] +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown 0.15.2", +] + [[package]] name = "indicatif" -version = "0.17.3" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width", + "unicode-width 0.2.0", + "web-time", ] [[package]] name = "inotify" -version = "0.9.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "inotify-sys", "libc", ] @@ -1109,54 +1242,59 @@ dependencies = [ ] [[package]] -name = "io-lifetimes" -version = "1.0.11" +name = "is_terminal_polyfill" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", + "either", ] [[package]] name = "itertools" -version = "0.12.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] name = "keccak" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ "cpufeatures", ] [[package]] name = "kqueue" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c8fc60ba15bf51257aa9807a48a61013db043fcf3a78cb0d916e8e396dcad98" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" dependencies = [ "kqueue-sys", "libc", @@ -1164,9 +1302,9 @@ dependencies = [ [[package]] name = "kqueue-sys" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8367585489f01bc55dd27404dcf56b95e6da061a256a666ab23be9ba96a2e587" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" dependencies = [ "bitflags 1.3.2", "libc", @@ -1174,84 +1312,102 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.152" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "winapi", + "windows-targets 0.48.5", ] [[package]] name = "libm" -version = "0.2.7" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.9.0", + "libc", + "redox_syscall", +] [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" -version = "0.4.17" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" + +[[package]] +name = "lru" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "cfg-if", + "hashbrown 0.15.2", ] [[package]] name = "lscolors" -version = "0.16.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b209ec3976527806024406fe765474b9a1750a0ed4b8f0372364741f50e7b" +checksum = "61183da5de8ba09a58e330d55e5ea796539d8443bd00fdeb863eac39724aa4ab" dependencies = [ + "aho-corasick", "nu-ansi-term", ] -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - [[package]] name = "md-5" version = "0.10.6" @@ -1264,15 +1420,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.9.0" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaba38d7abf1d4cca21cc89e932e542ba2b9258664d2a9ef0e61512039c9375" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" dependencies = [ "libc", ] @@ -1285,33 +1441,34 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "0.8.10" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", "log", - "wasi", - "windows-sys 0.48.0", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", ] [[package]] name = "nix" -version = "0.27.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.9.0", "cfg-if", + "cfg_aliases", "libc", ] @@ -1325,68 +1482,116 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "notify" -version = "6.0.1" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5738a2795d57ea20abec2d6d76c6081186709c0024187cd5977265eda6598b51" +checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" dependencies = [ - "bitflags 1.3.2", - "crossbeam-channel", + "bitflags 2.9.0", "filetime", "fsevent-sys", "inotify", "kqueue", "libc", + "log", "mio", + "notify-types", "walkdir", - "windows-sys 0.45.0", + "windows-sys 0.59.0", ] +[[package]] +name = "notify-types" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" + [[package]] name = "nu-ansi-term" -version = "0.49.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c073d3c1930d0751774acf49e66653acecb416c3a54c6ec095a9b11caddb5a68" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", + "rand 0.8.5", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] +[[package]] +name = "num-modular" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a5fe11d4135c3bcdf3a95b18b194afa9608a5f6ff034f5d857bc9a27fb0119" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-prime" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e238432a7881ec7164503ccc516c014bf009be7984cde1ba56837862543bdec3" +dependencies = [ + "bitvec", + "either", + "lru", + "num-bigint", + "num-integer", + "num-modular", + "num-traits", + "rand 0.8.5", +] + [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] @@ -1399,9 +1604,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "onig" @@ -1427,28 +1632,28 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.6.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" dependencies = [ "dlv-list", - "hashbrown", + "hashbrown 0.14.5", ] [[package]] name = "os_display" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6229bad892b46b0dcfaaeb18ad0d2e56400f5aaea05b768bde96e73676cf75" +checksum = "ad5fd71b79026fb918650dde6d125000a233764f1c2f1659a1c71118e33ea08f" dependencies = [ - "unicode-width", + "unicode-width 0.2.0", ] [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1456,47 +1661,51 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.0", + "windows-targets 0.52.6", ] [[package]] -name = "parse_datetime" -version = "0.5.0" +name = "parse-zoneinfo" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bbf4e25b13841080e018a1e666358adfe5e39b6d353f986ca5091c210b586a1" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" dependencies = [ - "chrono", "regex", ] [[package]] -name = "peeking_take_while" -version = "0.1.2" +name = "parse_datetime" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +checksum = "2fd3830b49ee3a0dcc8fdfadc68c6354c97d00101ac1cac5b2eee25d35c42066" +dependencies = [ + "chrono", + "nom 8.0.0", + "regex", +] [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", "phf_shared", @@ -1504,28 +1713,28 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -1535,15 +1744,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "platform-info" -version = "2.0.2" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6259c4860e53bf665016f1b2f46a8859cadfa717581dc9d597ae4069de6300f" +checksum = "7539aeb3fdd8cb4f6a331307cf71a1039cee75e94e8a71725b9484f4a0d9451a" dependencies = [ "libc", "winapi", @@ -1551,55 +1760,82 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "0.3.15" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15eb2c6e362923af47e13c23ca5afb859e83d54452c55b0b9ac763b8f7c1ac16" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ccf34da56fc294e7d4ccf69a85992b7dfb826b7cf57bac6a70bba3494cc08a" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] [[package]] name = "procfs" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.9.0", "hex", - "lazy_static", "procfs-core", - "rustix 0.38.30", + "rustix 0.38.44", ] [[package]] name = "procfs-core" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.9.0", "hex", ] @@ -1610,24 +1846,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] -name = "quickcheck" -version = "1.0.3" +name = "quote" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ - "env_logger", - "log", - "rand", + "proc-macro2", ] [[package]] -name = "quote" -version = "1.0.29" +name = "radium" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" -dependencies = [ - "proc-macro2", -] +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" @@ -1636,8 +1867,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -1647,7 +1888,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -1656,23 +1907,23 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] -name = "rand_pcg" -version = "0.3.1" +name = "rand_core" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "rand_core", + "getrandom 0.3.1", ] [[package]] name = "rayon" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -1680,9 +1931,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -1690,24 +1941,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", ] -[[package]] -name = "reference-counted-singleton" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bfbf25d7eb88ddcbb1ec3d755d0634da8f7657b2cb8b74089121409ab8228f" - [[package]] name = "regex" -version = "1.10.3" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -1717,9 +1962,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.4" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1728,112 +1973,119 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "relative-path" -version = "1.8.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf2521270932c3c7bed1a59151222bd7643c79310f2916f01925e1e16255698" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "rlimit" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3560f70f30a0f16d11d01ed078a07740fe6b489667abc7c7b029155d9f21c3d8" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" dependencies = [ "libc", ] [[package]] name = "roff" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" +checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3" [[package]] name = "rstest" -version = "0.18.2" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" +checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" dependencies = [ - "futures", "futures-timer", + "futures-util", "rstest_macros", "rustc_version", ] [[package]] name = "rstest_macros" -version = "0.18.2" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" +checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" dependencies = [ "cfg-if", "glob", + "proc-macro-crate", "proc-macro2", "quote", "regex", "relative-path", "rustc_version", - "syn 2.0.23", + "syn", "unicode-ident", ] [[package]] name = "rust-ini" -version = "0.19.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" +checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" dependencies = [ "cfg-if", "ordered-multimap", + "trim-in-place", ] [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.37.26" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f3f8f960ed3b5a59055428714943298bf3fa2d4a1d53135084e0544829d995" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "errno", - "io-lifetimes", "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", ] [[package]] name = "rustix" -version = "0.38.30" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "dade4812df5c384711475be5fcd8c162555352945401aed22a35bffeab61f657" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys 0.4.12", - "windows-sys 0.52.0", + "linux-raw-sys 0.9.4", + "windows-sys 0.59.0", ] +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + [[package]] name = "same-file" version = "1.0.6" @@ -1845,35 +2097,35 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "self_cell" -version = "1.0.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bf37232d3bb9a2c4e641ca2a11d83b5062066f88df7fed36c28772046d65ba" +checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749" [[package]] name = "selinux" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00576725d21b588213fbd4af84cd7e4cc4304e8e9bd6c0f5a1498a3e2ca6a51" +checksum = "e37f432dfe840521abd9a72fefdf88ed7ad0f43bbea7d9d1d3d80383e9f4ad13" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "libc", "once_cell", - "reference-counted-singleton", + "parking_lot", "selinux-sys", - "thiserror", + "thiserror 2.0.12", ] [[package]] name = "selinux-sys" -version = "0.6.2" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806d381649bb85347189d2350728817418138d11d738e2482cb644ec7f3c755d" +checksum = "280da3df1236da180be5ac50a893b26a1d3c49e3a44acb2d10d1f082523ff916" dependencies = [ "bindgen", "cc", @@ -1883,15 +2135,38 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.14" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "serde" -version = "1.0.147" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "sha1" @@ -1906,9 +2181,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -1927,15 +2202,15 @@ dependencies = [ [[package]] name = "shlex" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" dependencies = [ "libc", "signal-hook-registry", @@ -1943,9 +2218,9 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" dependencies = [ "libc", "mio", @@ -1954,24 +2229,30 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "siphasher" -version = "0.3.10" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -1987,37 +2268,37 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b187f0231d56fe41bfb12034819dd2bf336422a5866de41bc3fec4b2e3883e8" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "smawk" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "e02e925281e18ffd9d640e234264753c43edc62d64b2d4cf898f1bc5e75f3fc2" dependencies = [ "proc-macro2", "quote", @@ -2025,90 +2306,98 @@ dependencies = [ ] [[package]] -name = "syn" -version = "2.0.23" +name = "tap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.9.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ - "cfg-if", "fastrand", - "redox_syscall", - "rustix 0.38.30", - "windows-sys 0.52.0", + "getrandom 0.3.1", + "once_cell", + "rustix 1.0.1", + "windows-sys 0.59.0", ] [[package]] name = "terminal_size" -version = "0.2.6" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237" +checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" dependencies = [ - "rustix 0.37.26", - "windows-sys 0.48.0", + "rustix 1.0.1", + "windows-sys 0.59.0", ] [[package]] -name = "terminal_size" -version = "0.3.0" +name = "textwrap" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" dependencies = [ - "rustix 0.38.30", - "windows-sys 0.48.0", + "smawk", + "terminal_size", + "unicode-linebreak", + "unicode-width 0.2.0", ] [[package]] -name = "textwrap" -version = "0.16.0" +name = "thiserror" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "smawk", - "terminal_size 0.2.6", - "unicode-linebreak", - "unicode-width", + "thiserror-impl 1.0.69", ] [[package]] name = "thiserror" -version = "1.0.37" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ - "thiserror-impl", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "time" -version = "0.3.20" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ + "deranged", "itoa", "libc", + "num-conv", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -2116,16 +2405,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ + "num-conv", "time-core", ] @@ -2138,17 +2428,40 @@ dependencies = [ "crunchy", ] +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.22.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "trim-in-place" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" + [[package]] name = "typenum" -version = "1.15.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-linebreak" @@ -2158,37 +2471,73 @@ checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unindent" -version = "0.2.1" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "unty" +version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa30f5ea51ff7edfc797c6d3f9ec8cbd8cfedef5371766b7181d33977f4814f" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "utmp-classic" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e24c654e19afaa6b8f3877ece5d3bed849c2719c56f6752b18ca7da4fcc6e85a" +dependencies = [ + "cfg-if", + "libc", + "thiserror 1.0.69", + "time", + "utmp-classic-raw", + "zerocopy", +] + +[[package]] +name = "utmp-classic-raw" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "22c226537a3d6e01c440c1926ca0256dbee2d19b2229ede6fc4863a6493dd831" +dependencies = [ + "cfg-if", + "zerocopy", +] [[package]] name = "uu_arch" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "platform-info", @@ -2197,7 +2546,7 @@ dependencies = [ [[package]] name = "uu_base32" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2205,15 +2554,16 @@ dependencies = [ [[package]] name = "uu_base64" -version = "0.0.24" +version = "0.0.30" dependencies = [ + "clap", "uu_base32", "uucore", ] [[package]] name = "uu_basename" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2221,7 +2571,7 @@ dependencies = [ [[package]] name = "uu_basenc" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uu_base32", @@ -2230,29 +2580,30 @@ dependencies = [ [[package]] name = "uu_cat" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", + "memchr", "nix", - "thiserror", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_chcon" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "fts-sys", "libc", "selinux", - "thiserror", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_chgrp" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2260,7 +2611,7 @@ dependencies = [ [[package]] name = "uu_chmod" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -2269,7 +2620,7 @@ dependencies = [ [[package]] name = "uu_chown" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2277,24 +2628,26 @@ dependencies = [ [[package]] name = "uu_chroot" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_cksum" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "hex", + "regex", "uucore", ] [[package]] name = "uu_comm" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2302,13 +2655,14 @@ dependencies = [ [[package]] name = "uu_cp" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "exacl", "filetime", "indicatif", "libc", + "linux-raw-sys 0.9.4", "quick-error", "selinux", "uucore", @@ -2318,17 +2672,17 @@ dependencies = [ [[package]] name = "uu_csplit" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "regex", - "thiserror", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_cut" -version = "0.0.24" +version = "0.0.30" dependencies = [ "bstr", "clap", @@ -2338,41 +2692,43 @@ dependencies = [ [[package]] name = "uu_date" -version = "0.0.24" +version = "0.0.30" dependencies = [ "chrono", "clap", "libc", "parse_datetime", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_dd" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "gcd", "libc", "nix", "signal-hook", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_df" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "tempfile", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_dir" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uu_ls", @@ -2381,7 +2737,7 @@ dependencies = [ [[package]] name = "uu_dircolors" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2389,7 +2745,7 @@ dependencies = [ [[package]] name = "uu_dirname" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2397,18 +2753,19 @@ dependencies = [ [[package]] name = "uu_du" -version = "0.0.24" +version = "0.0.30" dependencies = [ "chrono", "clap", "glob", + "thiserror 2.0.12", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_echo" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2416,50 +2773,54 @@ dependencies = [ [[package]] name = "uu_env" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "nix", "rust-ini", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_expand" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_expr" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "num-bigint", "num-traits", "onig", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_factor" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "coz", + "num-bigint", + "num-prime", "num-traits", - "quickcheck", - "rand", + "rand 0.9.1", "smallvec", "uucore", ] [[package]] name = "uu_false" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2467,16 +2828,16 @@ dependencies = [ [[package]] name = "uu_fmt" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "unicode-width", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_fold" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2484,15 +2845,16 @@ dependencies = [ [[package]] name = "uu_groups" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_hashsum" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "hex", @@ -2503,16 +2865,17 @@ dependencies = [ [[package]] name = "uu_head" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "memchr", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_hostid" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -2521,17 +2884,18 @@ dependencies = [ [[package]] name = "uu_hostname" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", + "dns-lookup", "hostname", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_id" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "selinux", @@ -2540,27 +2904,29 @@ dependencies = [ [[package]] name = "uu_install" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "file_diff", "filetime", "libc", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_join" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "memchr", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_kill" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "nix", @@ -2569,7 +2935,7 @@ dependencies = [ [[package]] name = "uu_link" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2577,15 +2943,16 @@ dependencies = [ [[package]] name = "uu_ln" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_logname" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -2594,25 +2961,25 @@ dependencies = [ [[package]] name = "uu_ls" -version = "0.0.24" +version = "0.0.30" dependencies = [ + "ansi-width", "chrono", "clap", "glob", "hostname", "lscolors", "number_prefix", - "once_cell", "selinux", - "terminal_size 0.3.0", - "unicode-width", + "terminal_size", + "thiserror 2.0.12", "uucore", "uutils_term_grid", ] [[package]] name = "uu_mkdir" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2620,7 +2987,7 @@ dependencies = [ [[package]] name = "uu_mkfifo" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -2629,7 +2996,7 @@ dependencies = [ [[package]] name = "uu_mknod" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -2638,39 +3005,43 @@ dependencies = [ [[package]] name = "uu_mktemp" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "rand", + "rand 0.9.1", "tempfile", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_more" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "crossterm", "nix", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_mv" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "fs_extra", "indicatif", + "libc", + "thiserror 2.0.12", "uucore", + "windows-sys 0.59.0", ] [[package]] name = "uu_nice" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -2680,7 +3051,7 @@ dependencies = [ [[package]] name = "uu_nl" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "regex", @@ -2689,16 +3060,17 @@ dependencies = [ [[package]] name = "uu_nohup" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_nproc" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -2707,15 +3079,16 @@ dependencies = [ [[package]] name = "uu_numfmt" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_od" -version = "0.0.24" +version = "0.0.30" dependencies = [ "byteorder", "clap", @@ -2725,7 +3098,7 @@ dependencies = [ [[package]] name = "uu_paste" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2733,7 +3106,7 @@ dependencies = [ [[package]] name = "uu_pathchk" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -2742,7 +3115,7 @@ dependencies = [ [[package]] name = "uu_pinky" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2750,11 +3123,11 @@ dependencies = [ [[package]] name = "uu_pr" -version = "0.0.24" +version = "0.0.30" dependencies = [ "chrono", "clap", - "itertools", + "itertools 0.14.0", "quick-error", "regex", "uucore", @@ -2762,7 +3135,7 @@ dependencies = [ [[package]] name = "uu_printenv" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2770,7 +3143,7 @@ dependencies = [ [[package]] name = "uu_printf" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2778,16 +3151,17 @@ dependencies = [ [[package]] name = "uu_ptx" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "regex", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_pwd" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2795,7 +3169,7 @@ dependencies = [ [[package]] name = "uu_readlink" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2803,7 +3177,7 @@ dependencies = [ [[package]] name = "uu_realpath" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2811,18 +3185,17 @@ dependencies = [ [[package]] name = "uu_rm" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", "uucore", - "walkdir", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_rmdir" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -2831,95 +3204,98 @@ dependencies = [ [[package]] name = "uu_runcon" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", "selinux", - "thiserror", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_seq" -version = "0.0.24" +version = "0.0.30" dependencies = [ "bigdecimal", "clap", "num-bigint", "num-traits", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_shred" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", - "rand", + "rand 0.9.1", "uucore", ] [[package]] name = "uu_shuf" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "memchr", - "rand", - "rand_core", + "rand 0.9.1", + "rand_core 0.9.3", "uucore", ] [[package]] name = "uu_sleep" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "fundu", "uucore", ] [[package]] name = "uu_sort" -version = "0.0.24" +version = "0.0.30" dependencies = [ "binary-heap-plus", "clap", "compare", "ctrlc", "fnv", - "itertools", + "itertools 0.14.0", "memchr", - "rand", + "nix", + "rand 0.9.1", "rayon", "self_cell", "tempfile", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_split" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "memchr", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_stat" -version = "0.0.24" +version = "0.0.30" dependencies = [ + "chrono", "clap", "uucore", ] [[package]] name = "uu_stdbuf" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "tempfile", @@ -2929,7 +3305,7 @@ dependencies = [ [[package]] name = "uu_stdbuf_libstdbuf" -version = "0.0.24" +version = "0.0.30" dependencies = [ "cpp", "cpp_build", @@ -2938,7 +3314,7 @@ dependencies = [ [[package]] name = "uu_stty" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "nix", @@ -2947,7 +3323,7 @@ dependencies = [ [[package]] name = "uu_sum" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -2955,32 +3331,32 @@ dependencies = [ [[package]] name = "uu_sync" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", "nix", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_tac" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "memchr", "memmap2", "regex", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_tail" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "fundu", "libc", "memchr", "notify", @@ -2988,31 +3364,30 @@ dependencies = [ "same-file", "uucore", "winapi-util", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_tee" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "libc", + "nix", "uucore", ] [[package]] name = "uu_test" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", - "redox_syscall", "uucore", ] [[package]] name = "uu_timeout" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "libc", @@ -3022,28 +3397,29 @@ dependencies = [ [[package]] name = "uu_touch" -version = "0.0.24" +version = "0.0.30" dependencies = [ "chrono", "clap", "filetime", "parse_datetime", + "thiserror 2.0.12", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_tr" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "nom", + "nom 8.0.0", "uucore", ] [[package]] name = "uu_true" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -3051,7 +3427,7 @@ dependencies = [ [[package]] name = "uu_truncate" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -3059,15 +3435,16 @@ dependencies = [ [[package]] name = "uu_tsort" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_tty" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "nix", @@ -3076,7 +3453,7 @@ dependencies = [ [[package]] name = "uu_uname" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "platform-info", @@ -3085,16 +3462,17 @@ dependencies = [ [[package]] name = "uu_unexpand" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_uniq" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -3102,7 +3480,7 @@ dependencies = [ [[package]] name = "uu_unlink" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -3110,24 +3488,27 @@ dependencies = [ [[package]] name = "uu_uptime" -version = "0.0.24" +version = "0.0.30" dependencies = [ "chrono", "clap", + "thiserror 2.0.12", + "utmp-classic", "uucore", ] [[package]] name = "uu_users" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", + "utmp-classic", "uucore", ] [[package]] name = "uu_vdir" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uu_ls", @@ -3136,20 +3517,20 @@ dependencies = [ [[package]] name = "uu_wc" -version = "0.0.24" +version = "0.0.30" dependencies = [ "bytecount", "clap", "libc", "nix", - "thiserror", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_who" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", "uucore", @@ -3157,31 +3538,34 @@ dependencies = [ [[package]] name = "uu_whoami" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "libc", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_yes" -version = "0.0.24" +version = "0.0.30" dependencies = [ "clap", - "itertools", + "itertools 0.14.0", "nix", "uucore", ] [[package]] name = "uucore" -version = "0.0.24" +version = "0.0.30" dependencies = [ + "bigdecimal", "blake2b_simd", "blake3", + "chrono", + "chrono-tz", "clap", + "crc32fast", "data-encoding", "data-encoding-macro", "digest", @@ -3189,32 +3573,37 @@ dependencies = [ "dunce", "glob", "hex", - "itertools", + "iana-time-zone", + "itertools 0.14.0", "libc", "md-5", "memchr", "nix", - "once_cell", + "num-traits", + "number_prefix", "os_display", + "regex", + "selinux", "sha1", "sha2", "sha3", "sm3", "tempfile", - "thiserror", + "thiserror 2.0.12", "time", + "utmp-classic", "uucore_procs", "walkdir", "wild", "winapi-util", - "windows-sys 0.48.0", + "windows-sys 0.59.0", "xattr", "z85", ] [[package]] name = "uucore_procs" -version = "0.0.24" +version = "0.0.30" dependencies = [ "proc-macro2", "quote", @@ -3223,34 +3612,58 @@ dependencies = [ [[package]] name = "uuhelp_parser" -version = "0.0.24" +version = "0.0.30" [[package]] name = "uuid" -version = "1.2.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" + +[[package]] +name = "uutests" +version = "0.0.30" +dependencies = [ + "ctor", + "glob", + "libc", + "nix", + "pretty_assertions", + "rand 0.9.1", + "regex", + "rlimit", + "tempfile", + "time", + "uucore", + "xattr", +] [[package]] name = "uutils_term_grid" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b389452a568698688dda38802068378a16c15c4af9b153cdd99b65391292bbc7" +checksum = "fcba141ce511bad08e80b43f02976571072e1ff4286f7d628943efbd277c6361" dependencies = [ - "unicode-width", + "ansi-width", ] [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "virtue" +version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -3262,36 +3675,46 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.23", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3299,39 +3722,41 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] -name = "which" -version = "4.3.0" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ - "either", - "libc", - "once_cell", + "js-sys", + "wasm-bindgen", ] [[package]] name = "wild" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d01931a94d5a115a53f95292f51d316856b68a035618eb831bbba593a30b67" +checksum = "a3131afc8c575281e1e80f36ed6a092aa502c08b18ed7524e86fbbb12bb410e1" dependencies = [ "glob", ] @@ -3354,11 +3779,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -3368,234 +3793,307 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.45.0" +name = "windows-core" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247" dependencies = [ - "windows-targets 0.42.2", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] -name = "windows-sys" -version = "0.48.0" +name = "windows-implement" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" dependencies = [ - "windows-targets 0.48.0", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "windows-sys" -version = "0.52.0" +name = "windows-interface" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ - "windows-targets 0.52.0", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "windows-targets" -version = "0.42.2" +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-result" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-link", ] [[package]] -name = "windows-targets" +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows-targets 0.48.5", ] [[package]] -name = "windows-targets" +name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows-targets 0.52.6", ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" +name = "windows-sys" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" +name = "windows-targets" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.0" +name = "windows-targets" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] [[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" +name = "windows_aarch64_gnullvm" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" +name = "windows_aarch64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "windows_i686_gnu" -version = "0.42.2" +name = "windows_aarch64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] -name = "windows_i686_msvc" -version = "0.42.2" +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] -name = "windows_x86_64_gnu" -version = "0.52.0" +name = "windows_x86_64_gnullvm" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" +name = "windows_x86_64_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.0" +name = "windows_x86_64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +name = "winnow" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1" +dependencies = [ + "memchr", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" +name = "wit-bindgen-rt" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.9.0", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.52.0" +name = "wyz" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] [[package]] name = "xattr" -version = "1.3.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" dependencies = [ "libc", - "linux-raw-sys 0.4.12", - "rustix 0.38.30", + "rustix 1.0.1", ] [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "z85" -version = "3.0.5" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a599daf1b507819c1121f0bf87fa37eb19daac6aff3aefefd4e6e2e0f2020fc" +checksum = "9b3a41ce106832b4da1c065baa4c31cf640cf965fa1483816402b7f6b96f0a64" [[package]] -name = "zip" -version = "0.6.6" +name = "zerocopy" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" +dependencies = [ + "arbitrary", "crc32fast", "crossbeam-utils", "flate2", + "indexmap", + "memchr", + "zopfli", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", ] diff --git a/Cargo.toml b/Cargo.toml index 757cacf3492..a097f54ba10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,25 +1,26 @@ # coreutils (uutils) # * see the repository LICENSE, README, and CONTRIBUTING files for more information -# spell-checker:ignore (libs) bigdecimal datetime fundu gethostid kqueue libselinux mangen memmap procfs uuhelp +# spell-checker:ignore (libs) bigdecimal datetime serde bincode gethostid kqueue libselinux mangen memmap procfs uuhelp startswith constness expl [package] name = "coreutils" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "coreutils ~ GNU coreutils (updated); implemented as universal (cross-platform) utils, written in Rust" default-run = "coreutils" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils" readme = "README.md" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -rust-version = "1.70.0" -edition = "2021" - +rust-version = "1.85.0" build = "build.rs" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true + +[package.metadata.docs.rs] +all-features = true [features] default = ["feat_common_core"] @@ -30,6 +31,9 @@ windows = ["feat_os_windows"] ## project-specific feature shortcodes nightly = [] test_unimplemented = [] +expensive_tests = [] +# "test_risky_names" == enable tests that create problematic file names (would make a network share inaccessible to Windows, breaks SVN on Mac OS, etc.) +test_risky_names = [] # * only build `uudoc` when `--feature uudoc` is activated uudoc = ["zip", "dep:uuhelp_parser"] ## features @@ -46,6 +50,10 @@ feat_selinux = [ "cp/selinux", "id/selinux", "ls/selinux", + "mkdir/selinux", + "mkfifo/selinux", + "mknod/selinux", + "stat/selinux", "selinux", "feat_require_selinux", ] @@ -144,7 +152,8 @@ feat_os_macos = [ # "feat_require_unix_hostid", ] -# "feat_os_unix" == set of utilities which can be built/run on modern/usual *nix platforms +# "feat_os_unix" == set of utilities which can be built/run on modern/usual *nix platforms. +# Also used for targets binding to the "musl" library (ref: ) feat_os_unix = [ "feat_Tier1", # @@ -166,13 +175,6 @@ feat_os_unix_gnueabihf = [ "feat_require_unix_hostid", "feat_require_unix_utmpx", ] -# "feat_os_unix_musl" == set of utilities which can be built/run on targets binding to the "musl" library (ref: ) -feat_os_unix_musl = [ - "feat_Tier1", - # - "feat_require_unix", - "feat_require_unix_hostid", -] feat_os_unix_android = [ "feat_Tier1", # @@ -257,82 +259,97 @@ feat_os_windows_legacy = [ # * bypass/override ~ translate 'test' feature name to avoid dependency collision with rust core 'test' crate (o/w surfaces as compiler errors during testing) test = ["uu_test"] +[workspace.package] +authors = ["uutils developers"] +categories = ["command-line-utilities"] +edition = "2024" +homepage = "https://github.com/uutils/coreutils" +keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] +license = "MIT" +readme = "README.package.md" +version = "0.0.30" + [workspace.dependencies] +ansi-width = "0.1.0" bigdecimal = "0.4" binary-heap-plus = "0.5.0" -bstr = "1.9" -bytecount = "0.6.7" +bstr = "1.9.1" +bytecount = "0.6.8" byteorder = "1.5.0" -chrono = { version = "^0.4.32", default-features = false, features = [ +chrono = { version = "0.4.41", default-features = false, features = [ "std", "alloc", "clock", ] } -clap = { version = "4.4", features = ["wrap_help", "cargo"] } +chrono-tz = "0.10.0" +clap = { version = "4.5", features = ["wrap_help", "cargo"] } clap_complete = "4.4" clap_mangen = "0.2" compare = "0.1.0" coz = { version = "0.1.3" } -crossterm = ">=0.27.0" -ctrlc = { version = "3.4", features = ["termination"] } -exacl = "0.11.0" +crossterm = "0.29.0" +ctrlc = { version = "3.4.4", features = ["termination"] } +dns-lookup = { version = "2.0.4" } +exacl = "0.12.0" file_diff = "1.0.0" -filetime = "0.2" +filetime = "0.2.23" fnv = "1.0.7" fs_extra = "1.3.0" -fts-sys = "0.2" -fundu = "2.0.0" +fts-sys = "0.2.16" gcd = "2.3" glob = "0.3.1" -half = "2.3" -hostname = "0.3" -indicatif = "0.17" -itertools = "0.12.0" -libc = "0.2.152" -lscolors = { version = "0.16.0", default-features = false, features = [ +half = "2.4.1" +hostname = "0.4" +iana-time-zone = "0.1.57" +indicatif = "0.17.8" +itertools = "0.14.0" +libc = "0.2.172" +linux-raw-sys = "0.9" +lscolors = { version = "0.20.0", default-features = false, features = [ "gnu_legacy", ] } -memchr = "2" -memmap2 = "0.9" -nix = { version = "0.27", default-features = false } -nom = "7.1.3" -notify = { version = "=6.0.1", features = ["macos_kqueue"] } +memchr = "2.7.2" +memmap2 = "0.9.4" +nix = { version = "0.29", default-features = false } +nom = "8.0.0" +notify = { version = "=8.0.0", features = ["macos_kqueue"] } num-bigint = "0.4.4" -num-traits = "0.2.17" +num-prime = "0.4.4" +num-traits = "0.2.19" number_prefix = "0.4" -once_cell = "1.19.0" onig = { version = "~6.4", default-features = false } -parse_datetime = "0.5.0" +parse_datetime = "0.9.0" phf = "0.11.2" phf_codegen = "0.11.2" -platform-info = "2.0.2" +platform-info = "2.0.3" quick-error = "2.0.1" -rand = { version = "0.8", features = ["small_rng"] } -rand_core = "0.6" -rayon = "1.8" -redox_syscall = "0.4" -regex = "1.10.3" -rstest = "0.18.2" -rust-ini = "0.19.0" +rand = { version = "0.9.0", features = ["small_rng"] } +rand_core = "0.9.0" +rayon = "1.10" +regex = "1.10.4" +rstest = "0.25.0" +rust-ini = "0.21.0" same-file = "1.0.6" -self_cell = "1.0.3" -selinux = "0.4" +self_cell = "1.0.4" +selinux = "0.5.1" +selinux-sys = "0.6.14" signal-hook = "0.3.17" -smallvec = { version = "1.13", features = ["union"] } -tempfile = "3.9.0" -uutils_term_grid = "0.3" -terminal_size = "0.3.0" -textwrap = { version = "0.16.0", features = ["terminal_size"] } -thiserror = "1.0" -time = { version = "0.3" } -unicode-segmentation = "1.10.1" -unicode-width = "0.1.11" +smallvec = { version = "1.13.2", features = ["union"] } +tempfile = "3.15.0" +terminal_size = "0.4.0" +textwrap = { version = "0.16.1", features = ["terminal_size"] } +thiserror = "2.0.3" +time = { version = "0.3.36" } +unicode-segmentation = "1.11.0" +unicode-width = "0.2.0" utf-8 = "0.7.6" -walkdir = "2.4" -winapi-util = "0.1.6" -windows-sys = { version = "0.48.0", default-features = false } +utmp-classic = "0.1.6" +uutils_term_grid = "0.7" +walkdir = "2.5" +winapi-util = "0.1.8" +windows-sys = { version = "0.59.0", default-features = false } xattr = "1.3.1" -zip = { version = "0.6.6", default-features = false, features = ["deflate"] } +zip = { version = "2.2.2", default-features = false, features = ["deflate"] } hex = "0.4.3" md-5 = "0.10.6" @@ -340,18 +357,19 @@ sha1 = "0.10.6" sha2 = "0.10.8" sha3 = "0.10.8" blake2b_simd = "1.0.2" -blake3 = "1.5.0" +blake3 = "1.5.1" sm3 = "0.4.2" +crc32fast = "1.4.2" digest = "0.10.7" -uucore = { version = ">=0.0.19", package = "uucore", path = "src/uucore" } -uucore_procs = { version = ">=0.0.19", package = "uucore_procs", path = "src/uucore_procs" } -uu_ls = { version = ">=0.0.18", path = "src/uu/ls" } -uu_base32 = { version = ">=0.0.18", path = "src/uu/base32" } +uucore = { version = "0.0.30", package = "uucore", path = "src/uucore" } +uucore_procs = { version = "0.0.30", package = "uucore_procs", path = "src/uucore_procs" } +uu_ls = { version = "0.0.30", path = "src/uu/ls" } +uu_base32 = { version = "0.0.30", path = "src/uu/base32" } +uutests = { version = "0.0.30", package = "uutests", path = "tests/uutests/" } [dependencies] clap = { workspace = true } -once_cell = { workspace = true } uucore = { workspace = true } clap_complete = { workspace = true } clap_mangen = { workspace = true } @@ -363,109 +381,109 @@ zip = { workspace = true, optional = true } uuhelp_parser = { optional = true, version = ">=0.0.19", path = "src/uuhelp_parser" } # * uutils -uu_test = { optional = true, version = "0.0.24", package = "uu_test", path = "src/uu/test" } +uu_test = { optional = true, version = "0.0.30", package = "uu_test", path = "src/uu/test" } # -arch = { optional = true, version = "0.0.24", package = "uu_arch", path = "src/uu/arch" } -base32 = { optional = true, version = "0.0.24", package = "uu_base32", path = "src/uu/base32" } -base64 = { optional = true, version = "0.0.24", package = "uu_base64", path = "src/uu/base64" } -basename = { optional = true, version = "0.0.24", package = "uu_basename", path = "src/uu/basename" } -basenc = { optional = true, version = "0.0.24", package = "uu_basenc", path = "src/uu/basenc" } -cat = { optional = true, version = "0.0.24", package = "uu_cat", path = "src/uu/cat" } -chcon = { optional = true, version = "0.0.24", package = "uu_chcon", path = "src/uu/chcon" } -chgrp = { optional = true, version = "0.0.24", package = "uu_chgrp", path = "src/uu/chgrp" } -chmod = { optional = true, version = "0.0.24", package = "uu_chmod", path = "src/uu/chmod" } -chown = { optional = true, version = "0.0.24", package = "uu_chown", path = "src/uu/chown" } -chroot = { optional = true, version = "0.0.24", package = "uu_chroot", path = "src/uu/chroot" } -cksum = { optional = true, version = "0.0.24", package = "uu_cksum", path = "src/uu/cksum" } -comm = { optional = true, version = "0.0.24", package = "uu_comm", path = "src/uu/comm" } -cp = { optional = true, version = "0.0.24", package = "uu_cp", path = "src/uu/cp" } -csplit = { optional = true, version = "0.0.24", package = "uu_csplit", path = "src/uu/csplit" } -cut = { optional = true, version = "0.0.24", package = "uu_cut", path = "src/uu/cut" } -date = { optional = true, version = "0.0.24", package = "uu_date", path = "src/uu/date" } -dd = { optional = true, version = "0.0.24", package = "uu_dd", path = "src/uu/dd" } -df = { optional = true, version = "0.0.24", package = "uu_df", path = "src/uu/df" } -dir = { optional = true, version = "0.0.24", package = "uu_dir", path = "src/uu/dir" } -dircolors = { optional = true, version = "0.0.24", package = "uu_dircolors", path = "src/uu/dircolors" } -dirname = { optional = true, version = "0.0.24", package = "uu_dirname", path = "src/uu/dirname" } -du = { optional = true, version = "0.0.24", package = "uu_du", path = "src/uu/du" } -echo = { optional = true, version = "0.0.24", package = "uu_echo", path = "src/uu/echo" } -env = { optional = true, version = "0.0.24", package = "uu_env", path = "src/uu/env" } -expand = { optional = true, version = "0.0.24", package = "uu_expand", path = "src/uu/expand" } -expr = { optional = true, version = "0.0.24", package = "uu_expr", path = "src/uu/expr" } -factor = { optional = true, version = "0.0.24", package = "uu_factor", path = "src/uu/factor" } -false = { optional = true, version = "0.0.24", package = "uu_false", path = "src/uu/false" } -fmt = { optional = true, version = "0.0.24", package = "uu_fmt", path = "src/uu/fmt" } -fold = { optional = true, version = "0.0.24", package = "uu_fold", path = "src/uu/fold" } -groups = { optional = true, version = "0.0.24", package = "uu_groups", path = "src/uu/groups" } -hashsum = { optional = true, version = "0.0.24", package = "uu_hashsum", path = "src/uu/hashsum" } -head = { optional = true, version = "0.0.24", package = "uu_head", path = "src/uu/head" } -hostid = { optional = true, version = "0.0.24", package = "uu_hostid", path = "src/uu/hostid" } -hostname = { optional = true, version = "0.0.24", package = "uu_hostname", path = "src/uu/hostname" } -id = { optional = true, version = "0.0.24", package = "uu_id", path = "src/uu/id" } -install = { optional = true, version = "0.0.24", package = "uu_install", path = "src/uu/install" } -join = { optional = true, version = "0.0.24", package = "uu_join", path = "src/uu/join" } -kill = { optional = true, version = "0.0.24", package = "uu_kill", path = "src/uu/kill" } -link = { optional = true, version = "0.0.24", package = "uu_link", path = "src/uu/link" } -ln = { optional = true, version = "0.0.24", package = "uu_ln", path = "src/uu/ln" } -ls = { optional = true, version = "0.0.24", package = "uu_ls", path = "src/uu/ls" } -logname = { optional = true, version = "0.0.24", package = "uu_logname", path = "src/uu/logname" } -mkdir = { optional = true, version = "0.0.24", package = "uu_mkdir", path = "src/uu/mkdir" } -mkfifo = { optional = true, version = "0.0.24", package = "uu_mkfifo", path = "src/uu/mkfifo" } -mknod = { optional = true, version = "0.0.24", package = "uu_mknod", path = "src/uu/mknod" } -mktemp = { optional = true, version = "0.0.24", package = "uu_mktemp", path = "src/uu/mktemp" } -more = { optional = true, version = "0.0.24", package = "uu_more", path = "src/uu/more" } -mv = { optional = true, version = "0.0.24", package = "uu_mv", path = "src/uu/mv" } -nice = { optional = true, version = "0.0.24", package = "uu_nice", path = "src/uu/nice" } -nl = { optional = true, version = "0.0.24", package = "uu_nl", path = "src/uu/nl" } -nohup = { optional = true, version = "0.0.24", package = "uu_nohup", path = "src/uu/nohup" } -nproc = { optional = true, version = "0.0.24", package = "uu_nproc", path = "src/uu/nproc" } -numfmt = { optional = true, version = "0.0.24", package = "uu_numfmt", path = "src/uu/numfmt" } -od = { optional = true, version = "0.0.24", package = "uu_od", path = "src/uu/od" } -paste = { optional = true, version = "0.0.24", package = "uu_paste", path = "src/uu/paste" } -pathchk = { optional = true, version = "0.0.24", package = "uu_pathchk", path = "src/uu/pathchk" } -pinky = { optional = true, version = "0.0.24", package = "uu_pinky", path = "src/uu/pinky" } -pr = { optional = true, version = "0.0.24", package = "uu_pr", path = "src/uu/pr" } -printenv = { optional = true, version = "0.0.24", package = "uu_printenv", path = "src/uu/printenv" } -printf = { optional = true, version = "0.0.24", package = "uu_printf", path = "src/uu/printf" } -ptx = { optional = true, version = "0.0.24", package = "uu_ptx", path = "src/uu/ptx" } -pwd = { optional = true, version = "0.0.24", package = "uu_pwd", path = "src/uu/pwd" } -readlink = { optional = true, version = "0.0.24", package = "uu_readlink", path = "src/uu/readlink" } -realpath = { optional = true, version = "0.0.24", package = "uu_realpath", path = "src/uu/realpath" } -rm = { optional = true, version = "0.0.24", package = "uu_rm", path = "src/uu/rm" } -rmdir = { optional = true, version = "0.0.24", package = "uu_rmdir", path = "src/uu/rmdir" } -runcon = { optional = true, version = "0.0.24", package = "uu_runcon", path = "src/uu/runcon" } -seq = { optional = true, version = "0.0.24", package = "uu_seq", path = "src/uu/seq" } -shred = { optional = true, version = "0.0.24", package = "uu_shred", path = "src/uu/shred" } -shuf = { optional = true, version = "0.0.24", package = "uu_shuf", path = "src/uu/shuf" } -sleep = { optional = true, version = "0.0.24", package = "uu_sleep", path = "src/uu/sleep" } -sort = { optional = true, version = "0.0.24", package = "uu_sort", path = "src/uu/sort" } -split = { optional = true, version = "0.0.24", package = "uu_split", path = "src/uu/split" } -stat = { optional = true, version = "0.0.24", package = "uu_stat", path = "src/uu/stat" } -stdbuf = { optional = true, version = "0.0.24", package = "uu_stdbuf", path = "src/uu/stdbuf" } -stty = { optional = true, version = "0.0.24", package = "uu_stty", path = "src/uu/stty" } -sum = { optional = true, version = "0.0.24", package = "uu_sum", path = "src/uu/sum" } -sync = { optional = true, version = "0.0.24", package = "uu_sync", path = "src/uu/sync" } -tac = { optional = true, version = "0.0.24", package = "uu_tac", path = "src/uu/tac" } -tail = { optional = true, version = "0.0.24", package = "uu_tail", path = "src/uu/tail" } -tee = { optional = true, version = "0.0.24", package = "uu_tee", path = "src/uu/tee" } -timeout = { optional = true, version = "0.0.24", package = "uu_timeout", path = "src/uu/timeout" } -touch = { optional = true, version = "0.0.24", package = "uu_touch", path = "src/uu/touch" } -tr = { optional = true, version = "0.0.24", package = "uu_tr", path = "src/uu/tr" } -true = { optional = true, version = "0.0.24", package = "uu_true", path = "src/uu/true" } -truncate = { optional = true, version = "0.0.24", package = "uu_truncate", path = "src/uu/truncate" } -tsort = { optional = true, version = "0.0.24", package = "uu_tsort", path = "src/uu/tsort" } -tty = { optional = true, version = "0.0.24", package = "uu_tty", path = "src/uu/tty" } -uname = { optional = true, version = "0.0.24", package = "uu_uname", path = "src/uu/uname" } -unexpand = { optional = true, version = "0.0.24", package = "uu_unexpand", path = "src/uu/unexpand" } -uniq = { optional = true, version = "0.0.24", package = "uu_uniq", path = "src/uu/uniq" } -unlink = { optional = true, version = "0.0.24", package = "uu_unlink", path = "src/uu/unlink" } -uptime = { optional = true, version = "0.0.24", package = "uu_uptime", path = "src/uu/uptime" } -users = { optional = true, version = "0.0.24", package = "uu_users", path = "src/uu/users" } -vdir = { optional = true, version = "0.0.24", package = "uu_vdir", path = "src/uu/vdir" } -wc = { optional = true, version = "0.0.24", package = "uu_wc", path = "src/uu/wc" } -who = { optional = true, version = "0.0.24", package = "uu_who", path = "src/uu/who" } -whoami = { optional = true, version = "0.0.24", package = "uu_whoami", path = "src/uu/whoami" } -yes = { optional = true, version = "0.0.24", package = "uu_yes", path = "src/uu/yes" } +arch = { optional = true, version = "0.0.30", package = "uu_arch", path = "src/uu/arch" } +base32 = { optional = true, version = "0.0.30", package = "uu_base32", path = "src/uu/base32" } +base64 = { optional = true, version = "0.0.30", package = "uu_base64", path = "src/uu/base64" } +basename = { optional = true, version = "0.0.30", package = "uu_basename", path = "src/uu/basename" } +basenc = { optional = true, version = "0.0.30", package = "uu_basenc", path = "src/uu/basenc" } +cat = { optional = true, version = "0.0.30", package = "uu_cat", path = "src/uu/cat" } +chcon = { optional = true, version = "0.0.30", package = "uu_chcon", path = "src/uu/chcon" } +chgrp = { optional = true, version = "0.0.30", package = "uu_chgrp", path = "src/uu/chgrp" } +chmod = { optional = true, version = "0.0.30", package = "uu_chmod", path = "src/uu/chmod" } +chown = { optional = true, version = "0.0.30", package = "uu_chown", path = "src/uu/chown" } +chroot = { optional = true, version = "0.0.30", package = "uu_chroot", path = "src/uu/chroot" } +cksum = { optional = true, version = "0.0.30", package = "uu_cksum", path = "src/uu/cksum" } +comm = { optional = true, version = "0.0.30", package = "uu_comm", path = "src/uu/comm" } +cp = { optional = true, version = "0.0.30", package = "uu_cp", path = "src/uu/cp" } +csplit = { optional = true, version = "0.0.30", package = "uu_csplit", path = "src/uu/csplit" } +cut = { optional = true, version = "0.0.30", package = "uu_cut", path = "src/uu/cut" } +date = { optional = true, version = "0.0.30", package = "uu_date", path = "src/uu/date" } +dd = { optional = true, version = "0.0.30", package = "uu_dd", path = "src/uu/dd" } +df = { optional = true, version = "0.0.30", package = "uu_df", path = "src/uu/df" } +dir = { optional = true, version = "0.0.30", package = "uu_dir", path = "src/uu/dir" } +dircolors = { optional = true, version = "0.0.30", package = "uu_dircolors", path = "src/uu/dircolors" } +dirname = { optional = true, version = "0.0.30", package = "uu_dirname", path = "src/uu/dirname" } +du = { optional = true, version = "0.0.30", package = "uu_du", path = "src/uu/du" } +echo = { optional = true, version = "0.0.30", package = "uu_echo", path = "src/uu/echo" } +env = { optional = true, version = "0.0.30", package = "uu_env", path = "src/uu/env" } +expand = { optional = true, version = "0.0.30", package = "uu_expand", path = "src/uu/expand" } +expr = { optional = true, version = "0.0.30", package = "uu_expr", path = "src/uu/expr" } +factor = { optional = true, version = "0.0.30", package = "uu_factor", path = "src/uu/factor" } +false = { optional = true, version = "0.0.30", package = "uu_false", path = "src/uu/false" } +fmt = { optional = true, version = "0.0.30", package = "uu_fmt", path = "src/uu/fmt" } +fold = { optional = true, version = "0.0.30", package = "uu_fold", path = "src/uu/fold" } +groups = { optional = true, version = "0.0.30", package = "uu_groups", path = "src/uu/groups" } +hashsum = { optional = true, version = "0.0.30", package = "uu_hashsum", path = "src/uu/hashsum" } +head = { optional = true, version = "0.0.30", package = "uu_head", path = "src/uu/head" } +hostid = { optional = true, version = "0.0.30", package = "uu_hostid", path = "src/uu/hostid" } +hostname = { optional = true, version = "0.0.30", package = "uu_hostname", path = "src/uu/hostname" } +id = { optional = true, version = "0.0.30", package = "uu_id", path = "src/uu/id" } +install = { optional = true, version = "0.0.30", package = "uu_install", path = "src/uu/install" } +join = { optional = true, version = "0.0.30", package = "uu_join", path = "src/uu/join" } +kill = { optional = true, version = "0.0.30", package = "uu_kill", path = "src/uu/kill" } +link = { optional = true, version = "0.0.30", package = "uu_link", path = "src/uu/link" } +ln = { optional = true, version = "0.0.30", package = "uu_ln", path = "src/uu/ln" } +ls = { optional = true, version = "0.0.30", package = "uu_ls", path = "src/uu/ls" } +logname = { optional = true, version = "0.0.30", package = "uu_logname", path = "src/uu/logname" } +mkdir = { optional = true, version = "0.0.30", package = "uu_mkdir", path = "src/uu/mkdir" } +mkfifo = { optional = true, version = "0.0.30", package = "uu_mkfifo", path = "src/uu/mkfifo" } +mknod = { optional = true, version = "0.0.30", package = "uu_mknod", path = "src/uu/mknod" } +mktemp = { optional = true, version = "0.0.30", package = "uu_mktemp", path = "src/uu/mktemp" } +more = { optional = true, version = "0.0.30", package = "uu_more", path = "src/uu/more" } +mv = { optional = true, version = "0.0.30", package = "uu_mv", path = "src/uu/mv" } +nice = { optional = true, version = "0.0.30", package = "uu_nice", path = "src/uu/nice" } +nl = { optional = true, version = "0.0.30", package = "uu_nl", path = "src/uu/nl" } +nohup = { optional = true, version = "0.0.30", package = "uu_nohup", path = "src/uu/nohup" } +nproc = { optional = true, version = "0.0.30", package = "uu_nproc", path = "src/uu/nproc" } +numfmt = { optional = true, version = "0.0.30", package = "uu_numfmt", path = "src/uu/numfmt" } +od = { optional = true, version = "0.0.30", package = "uu_od", path = "src/uu/od" } +paste = { optional = true, version = "0.0.30", package = "uu_paste", path = "src/uu/paste" } +pathchk = { optional = true, version = "0.0.30", package = "uu_pathchk", path = "src/uu/pathchk" } +pinky = { optional = true, version = "0.0.30", package = "uu_pinky", path = "src/uu/pinky" } +pr = { optional = true, version = "0.0.30", package = "uu_pr", path = "src/uu/pr" } +printenv = { optional = true, version = "0.0.30", package = "uu_printenv", path = "src/uu/printenv" } +printf = { optional = true, version = "0.0.30", package = "uu_printf", path = "src/uu/printf" } +ptx = { optional = true, version = "0.0.30", package = "uu_ptx", path = "src/uu/ptx" } +pwd = { optional = true, version = "0.0.30", package = "uu_pwd", path = "src/uu/pwd" } +readlink = { optional = true, version = "0.0.30", package = "uu_readlink", path = "src/uu/readlink" } +realpath = { optional = true, version = "0.0.30", package = "uu_realpath", path = "src/uu/realpath" } +rm = { optional = true, version = "0.0.30", package = "uu_rm", path = "src/uu/rm" } +rmdir = { optional = true, version = "0.0.30", package = "uu_rmdir", path = "src/uu/rmdir" } +runcon = { optional = true, version = "0.0.30", package = "uu_runcon", path = "src/uu/runcon" } +seq = { optional = true, version = "0.0.30", package = "uu_seq", path = "src/uu/seq" } +shred = { optional = true, version = "0.0.30", package = "uu_shred", path = "src/uu/shred" } +shuf = { optional = true, version = "0.0.30", package = "uu_shuf", path = "src/uu/shuf" } +sleep = { optional = true, version = "0.0.30", package = "uu_sleep", path = "src/uu/sleep" } +sort = { optional = true, version = "0.0.30", package = "uu_sort", path = "src/uu/sort" } +split = { optional = true, version = "0.0.30", package = "uu_split", path = "src/uu/split" } +stat = { optional = true, version = "0.0.30", package = "uu_stat", path = "src/uu/stat" } +stdbuf = { optional = true, version = "0.0.30", package = "uu_stdbuf", path = "src/uu/stdbuf" } +stty = { optional = true, version = "0.0.30", package = "uu_stty", path = "src/uu/stty" } +sum = { optional = true, version = "0.0.30", package = "uu_sum", path = "src/uu/sum" } +sync = { optional = true, version = "0.0.30", package = "uu_sync", path = "src/uu/sync" } +tac = { optional = true, version = "0.0.30", package = "uu_tac", path = "src/uu/tac" } +tail = { optional = true, version = "0.0.30", package = "uu_tail", path = "src/uu/tail" } +tee = { optional = true, version = "0.0.30", package = "uu_tee", path = "src/uu/tee" } +timeout = { optional = true, version = "0.0.30", package = "uu_timeout", path = "src/uu/timeout" } +touch = { optional = true, version = "0.0.30", package = "uu_touch", path = "src/uu/touch" } +tr = { optional = true, version = "0.0.30", package = "uu_tr", path = "src/uu/tr" } +true = { optional = true, version = "0.0.30", package = "uu_true", path = "src/uu/true" } +truncate = { optional = true, version = "0.0.30", package = "uu_truncate", path = "src/uu/truncate" } +tsort = { optional = true, version = "0.0.30", package = "uu_tsort", path = "src/uu/tsort" } +tty = { optional = true, version = "0.0.30", package = "uu_tty", path = "src/uu/tty" } +uname = { optional = true, version = "0.0.30", package = "uu_uname", path = "src/uu/uname" } +unexpand = { optional = true, version = "0.0.30", package = "uu_unexpand", path = "src/uu/unexpand" } +uniq = { optional = true, version = "0.0.30", package = "uu_uniq", path = "src/uu/uniq" } +unlink = { optional = true, version = "0.0.30", package = "uu_unlink", path = "src/uu/unlink" } +uptime = { optional = true, version = "0.0.30", package = "uu_uptime", path = "src/uu/uptime" } +users = { optional = true, version = "0.0.30", package = "uu_users", path = "src/uu/users" } +vdir = { optional = true, version = "0.0.30", package = "uu_vdir", path = "src/uu/vdir" } +wc = { optional = true, version = "0.0.30", package = "uu_wc", path = "src/uu/wc" } +who = { optional = true, version = "0.0.30", package = "uu_who", path = "src/uu/who" } +whoami = { optional = true, version = "0.0.30", package = "uu_whoami", path = "src/uu/whoami" } +yes = { optional = true, version = "0.0.30", package = "uu_yes", path = "src/uu/yes" } # this breaks clippy linting with: "tests/by-util/test_factor_benches.rs: No such file or directory (os error 2)" # factor_benches = { optional = true, version = "0.0.0", package = "uu_factor_benches", path = "tests/benches/factor" } @@ -477,31 +495,46 @@ yes = { optional = true, version = "0.0.24", package = "uu_yes", path = "src/uu/ [dev-dependencies] chrono = { workspace = true } -conv = "0.3" filetime = { workspace = true } glob = { workspace = true } libc = { workspace = true } -pretty_assertions = "1" +num-prime = { workspace = true } +pretty_assertions = "1.4.0" rand = { workspace = true } regex = { workspace = true } -sha1 = { version = "0.10", features = ["std"] } +sha1 = { workspace = true, features = ["std"] } tempfile = { workspace = true } time = { workspace = true, features = ["local-offset"] } -unindent = "0.2" -uucore = { workspace = true, features = ["entries", "process", "signals"] } +unindent = "0.2.3" +uutests = { workspace = true } +uucore = { workspace = true, features = [ + "mode", + "entries", + "process", + "signals", + "utmpx", +] } walkdir = { workspace = true } -hex-literal = "0.4.1" +hex-literal = "1.0.0" rstest = { workspace = true } +ctor = "0.4.1" [target.'cfg(any(target_os = "linux", target_os = "android"))'.dev-dependencies] -procfs = { version = "0.16", default-features = false } -rlimit = "0.10.1" +procfs = { version = "0.17", default-features = false } [target.'cfg(unix)'.dev-dependencies] -nix = { workspace = true, features = ["process", "signal", "user"] } -rand_pcg = "0.3" +nix = { workspace = true, features = ["process", "signal", "user", "term"] } +rlimit = "0.10.1" xattr = { workspace = true } +# Specifically used in test_uptime::test_uptime_with_file_containing_valid_boot_time_utmpx_record +# to deserialize a utmpx struct into a binary file +[target.'cfg(all(target_family= "unix",not(target_os = "macos")))'.dev-dependencies] +serde = { version = "1.0.202", features = ["derive"] } +bincode = { version = "2.0.1", features = ["serde"] } +serde-big-array = "0.5.1" + + [build-dependencies] phf_codegen = { workspace = true } @@ -514,9 +547,9 @@ name = "uudoc" path = "src/bin/uudoc.rs" required-features = ["uudoc"] -# The default release profile. It contains all optimizations, without -# sacrificing debug info. With this profile (like in the standard -# release profile), the debug info and the stack traces will still be available. +# The default release profile. It contains all optimizations. +# With this profile (like in the standard release profile), +# the stack traces will still be available. [profile.release] lto = true @@ -532,3 +565,108 @@ inherits = "release" opt-level = "z" panic = "abort" strip = true + +# A release-like profile with debug info, useful for profiling. +# See https://github.com/mstange/samply . +[profile.profiling] +inherits = "release" +debug = true + +[lints.clippy] +multiple_crate_versions = "allow" +cargo_common_metadata = "allow" +uninlined_format_args = "allow" +missing_panics_doc = "allow" +# TODO remove when https://github.com/rust-lang/rust-clippy/issues/13774 is fixed +large_stack_arrays = "allow" + +use_self = "warn" +needless_pass_by_value = "warn" +semicolon_if_nothing_returned = "warn" +single_char_pattern = "warn" +explicit_iter_loop = "warn" +if_not_else = "warn" +manual_let_else = "warn" + +all = { level = "deny", priority = -1 } +cargo = { level = "warn", priority = -1 } +pedantic = { level = "deny", priority = -1 } + +# This is the linting configuration for all crates. +# Eventually the clippy settings from the `[lints]` section should be moved here. +# In order to use these, all crates have `[lints] workspace = true` section. +[workspace.lints.rust] +unused_qualifications = "warn" + +[workspace.lints.clippy] +# The counts were generated with this command: +# cargo clippy --all-targets --workspace --message-format=json --quiet \ +# | jq -r '.message.code.code | select(. != null and startswith("clippy::"))' \ +# | sort | uniq -c | sort -h -r +# +# TODO: +# remove large_stack_arrays when https://github.com/rust-lang/rust-clippy/issues/13774 is fixed +# +all = { level = "warn", priority = -1 } +cargo = { level = "warn", priority = -1 } +pedantic = { level = "warn", priority = -1 } +cargo_common_metadata = "allow" # 3240 +multiple_crate_versions = "allow" # 2314 +missing_errors_doc = "allow" # 1504 +missing_panics_doc = "allow" # 946 +must_use_candidate = "allow" # 322 +doc_markdown = "allow" # 267 +match_same_arms = "allow" # 212 +unnecessary_semicolon = "allow" # 156 +redundant_closure_for_method_calls = "allow" # 133 +cast_possible_truncation = "allow" # 118 +too_many_lines = "allow" # 81 +cast_possible_wrap = "allow" # 76 +trivially_copy_pass_by_ref = "allow" # 74 +cast_sign_loss = "allow" # 70 +struct_excessive_bools = "allow" # 68 +single_match_else = "allow" # 66 +redundant_else = "allow" # 58 +map_unwrap_or = "allow" # 54 +cast_precision_loss = "allow" # 52 +unnested_or_patterns = "allow" # 40 +inefficient_to_string = "allow" # 38 +unnecessary_wraps = "allow" # 37 +cast_lossless = "allow" # 33 +ignored_unit_patterns = "allow" # 29 +needless_continue = "allow" # 28 +items_after_statements = "allow" # 22 +similar_names = "allow" # 20 +wildcard_imports = "allow" # 18 +used_underscore_binding = "allow" # 16 +large_stack_arrays = "allow" # 14 +float_cmp = "allow" # 12 +# semicolon_if_nothing_returned = "allow" # 9 +used_underscore_items = "allow" # 8 +return_self_not_must_use = "allow" # 8 +needless_pass_by_value = "allow" # 8 +# manual_let_else = "allow" # 8 +# needless_raw_string_hashes = "allow" # 7 +match_on_vec_items = "allow" # 6 +inline_always = "allow" # 6 +# format_push_string = "allow" # 6 +fn_params_excessive_bools = "allow" # 6 +# single_char_pattern = "allow" # 4 +# ptr_cast_constness = "allow" # 4 +# match_wildcard_for_single_variants = "allow" # 4 +# manual_is_variant_and = "allow" # 4 +# explicit_deref_methods = "allow" # 4 +# enum_glob_use = "allow" # 3 +# unnecessary_literal_bound = "allow" # 2 +# stable_sort_primitive = "allow" # 2 +should_panic_without_expect = "allow" # 2 +# ptr_as_ptr = "allow" # 2 +# needless_for_each = "allow" # 2 +if_not_else = "allow" # 2 +expl_impl_clone_on_copy = "allow" # 2 +# cloned_instead_of_copied = "allow" # 2 +# borrow_as_ptr = "allow" # 2 +bool_to_int_with_if = "allow" # 2 +# ref_as_ptr = "allow" # 2 +# unreadable_literal = "allow" # 1 +uninlined_format_args = "allow" # ? diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 6f1de3b5476..c20d5acb7e8 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,4 +1,4 @@ - + # Setting up your local development environment @@ -241,6 +241,8 @@ DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl Note that GNU test suite relies on individual utilities (not the multicall binary). +You also need to install [quilt](https://savannah.nongnu.org/projects/quilt), a tool used to manage a stack of patches for modifying GNU tests. + On FreeBSD, you need to install packages for GNU coreutils and sed (used in shell scripts instead of system commands): ```shell @@ -251,13 +253,11 @@ pkg install coreutils gsed Code coverage report can be generated using [grcov](https://github.com/mozilla/grcov). -### Using Nightly Rust - To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-a-rust-project) coverage report ```shell export CARGO_INCREMENTAL=0 -export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" +export RUSTFLAGS="-Cinstrument-coverage -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" export RUSTDOCFLAGS="-Cpanic=abort" cargo build # e.g., --features feat_os_unix cargo test # e.g., --features feat_os_unix test_pathchk @@ -267,11 +267,6 @@ grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existin if changes are not reflected in the report then run `cargo clean` and run the above commands. -### Using Stable Rust - -If you are using stable version of Rust that doesn't enable code coverage instrumentation by default -then add `-Z-Zinstrument-coverage` flag to `RUSTFLAGS` env variable specified above. - ## Tips for setting up on Mac ### C Compiler and linker @@ -335,3 +330,13 @@ Otherwise please follow [this guide](https://learn.microsoft.com/en-us/windows/d If you have used [Git for Windows](https://gitforwindows.org) to install `git` on you Windows system you might already have some GNU core utilities installed as part of "GNU Bash" included in Git for Windows package, but it is not a complete package. [This article](https://gist.github.com/evanwill/0207876c3243bbb6863e65ec5dc3f058) provides instruction on how to add more to it. Alternatively you can install [Cygwin](https://www.cygwin.com) and/or use [WSL2](https://learn.microsoft.com/en-us/windows/wsl/compare-versions#whats-new-in-wsl-2) to get access to all GNU core utilities on Windows. + +# Preparing a new release + +1. Modify `util/update-version.sh` (FROM & TO) and run it +1. Submit a new PR with these changes and wait for it to be merged +1. Tag the new release `git tag -a X.Y.Z` and `git push --tags` +1. Once the CI is green, a new release will be automatically created in draft mode. + Reuse this release and make sure that assets have been added. +1. Write the release notes (it takes time) following previous examples +1. Run `util/publish.sh --do-it` to publish the new release to crates.io diff --git a/GNUmakefile b/GNUmakefile index ad2d38081f3..f46126a82f5 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -3,11 +3,19 @@ # Config options PROFILE ?= debug MULTICALL ?= n +COMPLETIONS ?= y +MANPAGES ?= y INSTALL ?= install ifneq (,$(filter install, $(MAKECMDGOALS))) override PROFILE:=release endif +# Needed for the foreach loops to split each loop into a separate command +define newline + + +endef + PROFILE_CMD := ifeq ($(PROFILE),release) PROFILE_CMD = --release @@ -33,23 +41,32 @@ PROG_PREFIX ?= # This won't support any directory with spaces in its name, but you can just # make a symlink without spaces that points to the directory. BASEDIR ?= $(shell pwd) +ifdef CARGO_TARGET_DIR +BUILDDIR := $(CARGO_TARGET_DIR)/${PROFILE} +else BUILDDIR := $(BASEDIR)/target/${PROFILE} +endif PKG_BUILDDIR := $(BUILDDIR)/deps DOCSDIR := $(BASEDIR)/docs BUSYBOX_ROOT := $(BASEDIR)/tmp -BUSYBOX_VER := 1.35.0 +BUSYBOX_VER := 1.36.1 BUSYBOX_SRC := $(BUSYBOX_ROOT)/busybox-$(BUSYBOX_VER) TOYBOX_ROOT := $(BASEDIR)/tmp -TOYBOX_VER := 0.8.8 +TOYBOX_VER := 0.8.12 TOYBOX_SRC := $(TOYBOX_ROOT)/toybox-$(TOYBOX_VER) -ifeq ($(SELINUX_ENABLED),) - SELINUX_ENABLED := 0 + +ifdef SELINUX_ENABLED + override SELINUX_ENABLED := 0 +# Now check if we should enable it (only on non-Windows) ifneq ($(OS),Windows_NT) - ifeq ($(shell /sbin/selinuxenabled 2>/dev/null ; echo $$?),0) - SELINUX_ENABLED := 1 + ifeq ($(shell if [ -x /sbin/selinuxenabled ] && /sbin/selinuxenabled 2>/dev/null; then echo 0; else echo 1; fi),0) + override SELINUX_ENABLED := 1 +$(info /sbin/selinuxenabled successful) + else +$(info SELINUX_ENABLED=1 but /sbin/selinuxenabled failed) endif endif endif @@ -164,9 +181,7 @@ SELINUX_PROGS := \ ifneq ($(OS),Windows_NT) PROGS := $(PROGS) $(UNIX_PROGS) -endif - -ifeq ($(SELINUX_ENABLED),1) +# Build the selinux command even if not on the system PROGS := $(PROGS) $(SELINUX_PROGS) endif @@ -221,6 +236,7 @@ TEST_PROGS := \ rmdir \ runcon \ seq \ + sleep \ sort \ split \ stat \ @@ -251,7 +267,8 @@ TEST_NO_FAIL_FAST :=--no-fail-fast TEST_SPEC_FEATURE := test_unimplemented else ifeq ($(SELINUX_ENABLED),1) TEST_NO_FAIL_FAST := -TEST_SPEC_FEATURE := feat_selinux +TEST_SPEC_FEATURE := selinux +BUILD_SPEC_FEATURE := selinux endif define TEST_BUSYBOX @@ -275,11 +292,15 @@ use_default := 1 build-pkgs: ifneq (${MULTICALL}, y) +ifdef BUILD_SPEC_FEATURE + ${CARGO} build ${CARGOFLAGS} --features "$(BUILD_SPEC_FEATURE)" ${PROFILE_CMD} $(foreach pkg,$(EXES),-p uu_$(pkg)) +else ${CARGO} build ${CARGOFLAGS} ${PROFILE_CMD} $(foreach pkg,$(EXES),-p uu_$(pkg)) endif +endif build-coreutils: - ${CARGO} build ${CARGOFLAGS} --features "${EXES}" ${PROFILE_CMD} --no-default-features + ${CARGO} build ${CARGOFLAGS} --features "${EXES} $(BUILD_SPEC_FEATURE)" ${PROFILE_CMD} --no-default-features build: build-coreutils build-pkgs @@ -336,45 +357,58 @@ clean: distclean: clean $(CARGO) clean $(CARGOFLAGS) && $(CARGO) update $(CARGOFLAGS) +ifeq ($(MANPAGES),y) manpages: build-coreutils mkdir -p $(BUILDDIR)/man/ $(foreach prog, $(INSTALLEES), \ - $(BUILDDIR)/coreutils manpage $(prog) > $(BUILDDIR)/man/$(PROG_PREFIX)$(prog).1; \ + $(BUILDDIR)/coreutils manpage $(prog) > $(BUILDDIR)/man/$(PROG_PREFIX)$(prog).1 $(newline) \ ) +install-manpages: manpages + mkdir -p $(DESTDIR)$(DATAROOTDIR)/man/man1 + $(foreach prog, $(INSTALLEES), \ + $(INSTALL) $(BUILDDIR)/man/$(PROG_PREFIX)$(prog).1 $(DESTDIR)$(DATAROOTDIR)/man/man1/ $(newline) \ + ) +else +install-manpages: +endif + +ifeq ($(COMPLETIONS),y) completions: build-coreutils mkdir -p $(BUILDDIR)/completions/zsh $(BUILDDIR)/completions/bash $(BUILDDIR)/completions/fish $(foreach prog, $(INSTALLEES), \ - $(BUILDDIR)/coreutils completion $(prog) zsh > $(BUILDDIR)/completions/zsh/_$(PROG_PREFIX)$(prog); \ - $(BUILDDIR)/coreutils completion $(prog) bash > $(BUILDDIR)/completions/bash/$(PROG_PREFIX)$(prog); \ - $(BUILDDIR)/coreutils completion $(prog) fish > $(BUILDDIR)/completions/fish/$(PROG_PREFIX)$(prog).fish; \ + $(BUILDDIR)/coreutils completion $(prog) zsh > $(BUILDDIR)/completions/zsh/_$(PROG_PREFIX)$(prog) $(newline) \ + $(BUILDDIR)/coreutils completion $(prog) bash > $(BUILDDIR)/completions/bash/$(PROG_PREFIX)$(prog) $(newline) \ + $(BUILDDIR)/coreutils completion $(prog) fish > $(BUILDDIR)/completions/fish/$(PROG_PREFIX)$(prog).fish $(newline) \ ) -install: build manpages completions +install-completions: completions + mkdir -p $(DESTDIR)$(DATAROOTDIR)/zsh/site-functions + mkdir -p $(DESTDIR)$(DATAROOTDIR)/bash-completion/completions + mkdir -p $(DESTDIR)$(DATAROOTDIR)/fish/vendor_completions.d + $(foreach prog, $(INSTALLEES), \ + $(INSTALL) $(BUILDDIR)/completions/zsh/_$(PROG_PREFIX)$(prog) $(DESTDIR)$(DATAROOTDIR)/zsh/site-functions/ $(newline) \ + $(INSTALL) $(BUILDDIR)/completions/bash/$(PROG_PREFIX)$(prog) $(DESTDIR)$(DATAROOTDIR)/bash-completion/completions/ $(newline) \ + $(INSTALL) $(BUILDDIR)/completions/fish/$(PROG_PREFIX)$(prog).fish $(DESTDIR)$(DATAROOTDIR)/fish/vendor_completions.d/ $(newline) \ + ) +else +install-completions: +endif + +install: build install-manpages install-completions mkdir -p $(INSTALLDIR_BIN) ifeq (${MULTICALL}, y) $(INSTALL) $(BUILDDIR)/coreutils $(INSTALLDIR_BIN)/$(PROG_PREFIX)coreutils - cd $(INSTALLDIR_BIN) && $(foreach prog, $(filter-out coreutils, $(INSTALLEES)), \ - ln -fs $(PROG_PREFIX)coreutils $(PROG_PREFIX)$(prog) &&) : + $(foreach prog, $(filter-out coreutils, $(INSTALLEES)), \ + cd $(INSTALLDIR_BIN) && ln -fs $(PROG_PREFIX)coreutils $(PROG_PREFIX)$(prog) $(newline) \ + ) $(if $(findstring test,$(INSTALLEES)), cd $(INSTALLDIR_BIN) && ln -fs $(PROG_PREFIX)coreutils $(PROG_PREFIX)[) else $(foreach prog, $(INSTALLEES), \ - $(INSTALL) $(BUILDDIR)/$(prog) $(INSTALLDIR_BIN)/$(PROG_PREFIX)$(prog);) + $(INSTALL) $(BUILDDIR)/$(prog) $(INSTALLDIR_BIN)/$(PROG_PREFIX)$(prog) $(newline) \ + ) $(if $(findstring test,$(INSTALLEES)), $(INSTALL) $(BUILDDIR)/test $(INSTALLDIR_BIN)/$(PROG_PREFIX)[) endif - mkdir -p $(DESTDIR)$(DATAROOTDIR)/man/man1 - $(foreach prog, $(INSTALLEES), \ - $(INSTALL) $(BUILDDIR)/man/$(PROG_PREFIX)$(prog).1 $(DESTDIR)$(DATAROOTDIR)/man/man1/; \ - ) - - mkdir -p $(DESTDIR)$(DATAROOTDIR)/zsh/site-functions - mkdir -p $(DESTDIR)$(DATAROOTDIR)/bash-completion/completions - mkdir -p $(DESTDIR)$(DATAROOTDIR)/fish/vendor_completions.d - $(foreach prog, $(INSTALLEES), \ - $(INSTALL) $(BUILDDIR)/completions/zsh/_$(PROG_PREFIX)$(prog) $(DESTDIR)$(DATAROOTDIR)/zsh/site-functions/; \ - $(INSTALL) $(BUILDDIR)/completions/bash/$(PROG_PREFIX)$(prog) $(DESTDIR)$(DATAROOTDIR)/bash-completion/completions/; \ - $(INSTALL) $(BUILDDIR)/completions/fish/$(PROG_PREFIX)$(prog).fish $(DESTDIR)$(DATAROOTDIR)/fish/vendor_completions.d/; \ - ) uninstall: ifeq (${MULTICALL}, y) diff --git a/README.md b/README.md index c8ca0d8a316..1d9a7ddd190 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ [![dependency status](https://deps.rs/repo/github/uutils/coreutils/status.svg)](https://deps.rs/repo/github/uutils/coreutils) [![CodeCov](https://codecov.io/gh/uutils/coreutils/branch/master/graph/badge.svg)](https://codecov.io/gh/uutils/coreutils) -![MSRV](https://img.shields.io/badge/MSRV-1.70.0-brightgreen) +![MSRV](https://img.shields.io/badge/MSRV-1.85.0-brightgreen) @@ -45,17 +45,16 @@ uutils aims to be a drop-in replacement for the GNU utils. Differences with GNU are treated as bugs. uutils aims to work on as many platforms as possible, to be able to use the same -utils on Linux, Mac, Windows and other platforms. This ensures, for example, +utils on Linux, macOS, Windows and other platforms. This ensures, for example, that scripts can be easily transferred between platforms.
## Documentation - uutils has both user and developer documentation available: -- [User Manual](https://uutils.github.io/coreutils/book/) -- [Developer Documentation](https://uutils.github.io/dev/coreutils/) (currently offline, you can use docs.rs in the meantime) +- [User Manual](https://uutils.github.io/coreutils/docs/) +- [Developer Documentation](https://docs.rs/crate/coreutils/) Both can also be generated locally, the instructions for that can be found in the [coreutils docs](https://github.com/uutils/uutils.github.io) repository. @@ -71,7 +70,7 @@ the [coreutils docs](https://github.com/uutils/uutils.github.io) repository. ### Rust Version uutils follows Rust's release channels and is tested against stable, beta and -nightly. The current Minimum Supported Rust Version (MSRV) is `1.70.0`. +nightly. The current Minimum Supported Rust Version (MSRV) is `1.85.0`. ## Building @@ -79,7 +78,7 @@ There are currently two methods to build the uutils binaries: either Cargo or GNU Make. > Building the full package, including all documentation, requires both Cargo -> and Gnu Make on a Unix platform. +> and GNU Make on a Unix platform. For either method, we first need to fetch the repository: @@ -224,6 +223,12 @@ Installing with `make` installs shell completions for all installed utilities for `bash`, `fish` and `zsh`. Completions for `elvish` and `powershell` can also be generated; See `Manually install shell completions`. +To skip installation of completions and manpages: + +```shell +make COMPLETIONS=n MANPAGES=n install +``` + ### Manually install shell completions The `coreutils` binary can generate completions for the `bash`, `elvish`, @@ -267,7 +272,7 @@ Make to uninstall. To uninstall uutils: ```shell -cargo uninstall uutils +cargo uninstall coreutils ``` ### Uninstall with GNU Make @@ -303,12 +308,12 @@ make PREFIX=/my/path uninstall Below is the evolution of how many GNU tests uutils passes. A more detailed breakdown of the GNU test results of the main branch can be found -[in the user manual](https://uutils.github.io/coreutils/book/test_coverage.html). +[in the user manual](https://uutils.github.io/coreutils/docs/test_coverage.html). See for the main meta bugs (many are missing). -![Evolution over time](https://github.com/uutils/coreutils-tracking/blob/main/gnu-results.png?raw=true) +![Evolution over time](https://github.com/uutils/coreutils-tracking/blob/main/gnu-results.svg?raw=true)
diff --git a/README.package.md b/README.package.md new file mode 100644 index 00000000000..355b153db28 --- /dev/null +++ b/README.package.md @@ -0,0 +1,31 @@ + + + +
+
+ +![uutils logo](docs/src/logo.svg) + +# uutils coreutils + +[![Crates.io](https://img.shields.io/crates/v/coreutils.svg)](https://crates.io/crates/coreutils) +[![Discord](https://img.shields.io/badge/discord-join-7289DA.svg?logo=discord&longCache=true&style=flat)](https://discord.gg/wQVJbvJ) +[![License](http://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/uutils/coreutils/blob/main/LICENSE) +[![dependency status](https://deps.rs/repo/github/uutils/coreutils/status.svg)](https://deps.rs/repo/github/uutils/coreutils) + +[![CodeCov](https://codecov.io/gh/uutils/coreutils/branch/master/graph/badge.svg)](https://codecov.io/gh/uutils/coreutils) +![MSRV](https://img.shields.io/badge/MSRV-1.70.0-brightgreen) + +
+ +--- + +
+ +This package is part of uutils coreutils. + +uutils coreutils is a cross-platform reimplementation of the GNU coreutils in +[Rust](http://www.rust-lang.org). + +This package does not have its specific `README.md`. + diff --git a/build.rs b/build.rs index bb4e2b53609..3b6aa3878d1 100644 --- a/build.rs +++ b/build.rs @@ -11,14 +11,18 @@ use std::io::Write; use std::path::Path; pub fn main() { - if let Ok(profile) = env::var("PROFILE") { - println!("cargo:rustc-cfg=build={profile:?}"); - } - const ENV_FEATURE_PREFIX: &str = "CARGO_FEATURE_"; const FEATURE_PREFIX: &str = "feat_"; const OVERRIDE_PREFIX: &str = "uu_"; + // Do not rebuild build script unless the script itself or the enabled features are modified + // See + println!("cargo:rerun-if-changed=build.rs"); + + if let Ok(profile) = env::var("PROFILE") { + println!("cargo:rustc-cfg=build={profile:?}"); + } + let out_dir = env::var("OUT_DIR").unwrap(); let mut crates = Vec::new(); @@ -29,8 +33,10 @@ pub fn main() { #[allow(clippy::match_same_arms)] match krate.as_ref() { "default" | "macos" | "unix" | "windows" | "selinux" | "zip" => continue, // common/standard feature names - "nightly" | "test_unimplemented" => continue, // crate-local custom features - "uudoc" => continue, // is not a utility + "nightly" | "test_unimplemented" | "expensive_tests" | "test_risky_names" => { + continue; + } // crate-local custom features + "uudoc" => continue, // is not a utility "test" => continue, // over-ridden with 'uu_test' to avoid collision with rust core crate 'test' s if s.starts_with(FEATURE_PREFIX) => continue, // crate feature sets _ => {} // util feature name @@ -46,6 +52,7 @@ pub fn main() { "type UtilityMap = phf::OrderedMap<&'static str, (fn(T) -> i32, fn() -> Command)>;\n\ \n\ #[allow(clippy::too_many_lines)] + #[allow(clippy::unreadable_literal)] fn util_map() -> UtilityMap {\n" .as_bytes(), ) diff --git a/deny.toml b/deny.toml index d7c04ad2d5b..1ae92b14a72 100644 --- a/deny.toml +++ b/deny.toml @@ -6,10 +6,8 @@ [advisories] db-path = "~/.cargo/advisory-db" db-urls = ["https://github.com/rustsec/advisory-db"] -vulnerability = "warn" -unmaintained = "warn" +version = 2 yanked = "warn" -notice = "warn" ignore = [ #"RUSTSEC-0000-0000", ] @@ -18,20 +16,18 @@ ignore = [ # More documentation for the licenses section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html [licenses] -unlicensed = "deny" +version = 2 allow = [ "MIT", "Apache-2.0", "ISC", "BSD-2-Clause", - "BSD-2-Clause-FreeBSD", "BSD-3-Clause", + "BSL-1.0", "CC0-1.0", - "Unicode-DFS-2016", + "Unicode-3.0", + "Zlib", ] -copyleft = "deny" -allow-osi-fsf-free = "neither" -default = "deny" confidence-threshold = 0.8 [[licenses.clarify]] @@ -58,33 +54,13 @@ highlight = "all" # introduces it. # spell-checker: disable skip = [ - # rustix - { name = "linux-raw-sys", version = "0.3.8" }, - # terminal_size - { name = "rustix", version = "0.37.26" }, - # notify - { name = "windows-sys", version = "0.45.0" }, - # various crates + # dns-lookup { name = "windows-sys", version = "0.48.0" }, - # windows-sys - { name = "windows-targets", version = "0.42.2" }, + # mio, nu-ansi-term, socket2 + { name = "windows-sys", version = "0.52.0" }, # windows-sys { name = "windows-targets", version = "0.48.0" }, # windows-targets - { name = "windows_aarch64_gnullvm", version = "0.42.2" }, - # windows-targets - { name = "windows_aarch64_msvc", version = "0.42.2" }, - # windows-targets - { name = "windows_i686_gnu", version = "0.42.2" }, - # windows-targets - { name = "windows_i686_msvc", version = "0.42.2" }, - # windows-targets - { name = "windows_x86_64_gnu", version = "0.42.2" }, - # windows-targets - { name = "windows_x86_64_gnullvm", version = "0.42.2" }, - # windows-targets - { name = "windows_x86_64_msvc", version = "0.42.2" }, - # windows-targets { name = "windows_aarch64_gnullvm", version = "0.48.0" }, # windows-targets { name = "windows_aarch64_msvc", version = "0.48.0" }, @@ -98,12 +74,34 @@ skip = [ { name = "windows_x86_64_gnullvm", version = "0.48.0" }, # windows-targets { name = "windows_x86_64_msvc", version = "0.48.0" }, - # various crates - { name = "syn", version = "1.0.109" }, - # various crates + # kqueue-sys, onig { name = "bitflags", version = "1.3.2" }, - # clap_builder, textwrap - { name = "terminal_size", version = "0.2.6" }, + # ansi-width + { name = "unicode-width", version = "0.1.13" }, + # filedescriptor, utmp-classic + { name = "thiserror", version = "1.0.69" }, + # thiserror + { name = "thiserror-impl", version = "1.0.69" }, + # bindgen + { name = "itertools", version = "0.13.0" }, + # ordered-multimap + { name = "hashbrown", version = "0.14.5" }, + # cexpr (via bindgen) + { name = "nom", version = "7.1.3" }, + # const-random-macro, rand_core + { name = "getrandom", version = "0.2.15" }, + # getrandom, mio + { name = "wasi", version = "0.11.0+wasi-snapshot-preview1" }, + # num-bigint, num-prime, phf_generator + { name = "rand", version = "0.8.5" }, + # rand + { name = "rand_chacha", version = "0.3.1" }, + # rand + { name = "rand_core", version = "0.6.4" }, + # crossterm, procfs, terminal_size + { name = "rustix", version = "0.38.43" }, + # rustix + { name = "linux-raw-sys", version = "0.4.15" }, ] # spell-checker: enable diff --git a/docs/compiles_table.py b/docs/compiles_table.py index 83c98bed1de..884b0bc39e1 100644 --- a/docs/compiles_table.py +++ b/docs/compiles_table.py @@ -10,7 +10,7 @@ # third party dependencies from tqdm import tqdm -# spell-checker:ignore (libs) tqdm imap ; (shell/mac) xcrun ; (vars) nargs +# spell-checker:ignore (libs) tqdm imap ; (shell/mac) xcrun ; (vars) nargs retcode csvfile BINS_PATH = Path("../src/uu") CACHE_PATH = Path("compiles_table.csv") diff --git a/docs/src/CODE_OF_CONDUCT.md b/docs/src/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..ce326a1ee03 --- /dev/null +++ b/docs/src/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ + + +{{ #include ../../CODE_OF_CONDUCT.md }} diff --git a/docs/src/contributing.md b/docs/src/CONTRIBUTING.md similarity index 100% rename from docs/src/contributing.md rename to docs/src/CONTRIBUTING.md diff --git a/docs/src/DEVELOPMENT.md b/docs/src/DEVELOPMENT.md new file mode 100644 index 00000000000..580cecf0855 --- /dev/null +++ b/docs/src/DEVELOPMENT.md @@ -0,0 +1,3 @@ + + +{{ #include ../../DEVELOPMENT.md }} diff --git a/docs/src/extensions.md b/docs/src/extensions.md index 79746498f2d..6cd7b8b443d 100644 --- a/docs/src/extensions.md +++ b/docs/src/extensions.md @@ -75,7 +75,29 @@ number of spaces representing a tab when determining the line length. GNU `ls` provides two ways to use a long listing format: `-l` and `--format=long`. We support a third way: `--long`. +GNU `ls --sort=VALUE` only supports special non-default sort orders. +We support `--sort=name`, which makes it possible to override an earlier value. + ## `du` `du` allows `birth` and `creation` as values for the `--time` argument to show the creation time. It also provides a `-v`/`--verbose` flag. + +## `id` + +`id` has three additional flags: +* `-P` displays the id as a password file entry +* `-p` makes the output human-readable +* `-A` displays the process audit user ID + +## `uptime` + +Similar to the proc-ps implementation and unlike GNU/Coreutils, `uptime` provides `-s`/`--since` to show since when the system is up. + +## `base32/base64/basenc` + +Just like on macOS, `base32/base64/basenc` provides `-D` to decode data. + +## `shred` + +The number of random passes is deterministic in both GNU and uutils. However, uutils `shred` computes the number of random passes in a simplified way, specifically `max(3, x / 10)`, which is very close but not identical to the number of random passes that GNU would do. This also satisfies an expectation that reasonable users might have, namely that the number of random passes increases monotonically with the number of passes overall; GNU `shred` violates this assumption. diff --git a/docs/src/installation.md b/docs/src/installation.md index dc631d240ca..856ca9d22f5 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -1,4 +1,4 @@ - + # Installation @@ -16,11 +16,11 @@ You can also [build uutils from source](build.md). ```shell # Linux -cargo install coreutils --features unix +cargo install coreutils --features unix --locked # MacOs -cargo install coreutils --features macos +cargo install coreutils --features macos --locked # Windows -cargo install coreutils --features windows +cargo install coreutils --features windows --locked ``` ## Linux @@ -53,6 +53,16 @@ apt install rust-coreutils export PATH=/usr/lib/cargo/bin/coreutils:$PATH ``` +### Fedora + +[![Fedora package](https://repology.org/badge/version-for-repo/fedora_rawhide/uutils-coreutils.svg)](https://packages.fedoraproject.org/pkgs/rust-coreutils/uutils-coreutils) + +```shell +dnf install uutils-coreutils +# To use it: +export PATH=/usr/libexec/uutils-coreutils:$PATH +``` + ### Gentoo [![Gentoo package](https://repology.org/badge/version-for-repo/gentoo/uutils-coreutils.svg)](https://packages.gentoo.org/packages/sys-apps/uutils-coreutils) @@ -89,9 +99,22 @@ nix-env -iA nixos.uutils-coreutils dnf install uutils-coreutils ``` +### RHEL/AlmaLinux/CENTOS Stream/Rocky Linux/EPEL 9 + +[![epel 9 package](https://repology.org/badge/version-for-repo/epel_9/uutils-coreutils.svg)](https://packages.fedoraproject.org/pkgs/rust-coreutils/uutils-coreutils/epel-9.html) + +```shell +# Install EPEL 9 - Specific For RHEL please check codeready-builder-for-rhel-9 First then install epel +dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm -y +# Install Core Utils +dnf install uutils-coreutils +# To use it: +export PATH=/usr/libexec/uutils-coreutils:$PATH +``` + ### Ubuntu -[![Ubuntu package](https://repology.org/badge/version-for-repo/ubuntu_23_04/uutils-coreutils.svg)](https://packages.ubuntu.com/source/lunar/rust-coreutils) +[![Ubuntu package](https://repology.org/badge/version-for-repo/ubuntu_25_04/uutils-coreutils.svg)](https://packages.ubuntu.com/source/plucky/rust-coreutils) ```shell apt install rust-coreutils @@ -151,12 +174,21 @@ scoop install uutils-coreutils conda install -c conda-forge uutils-coreutils ``` +### Yocto + +[Yocto recipe](https://github.com/openembedded/meta-openembedded/tree/master/meta-oe/recipes-core/uutils-coreutils) + +The uutils-coreutils recipe is provided as part of the meta-openembedded yocto layer. +Clone [poky](https://github.com/yoctoproject/poky) and [meta-openembedded](https://github.com/openembedded/meta-openembedded/tree/master), add +`meta-openembedded/meta-oe` as layer in your `build/conf/bblayers.conf` file, +and then either call `bitbake uutils-coreutils`, or use +`PREFERRED_PROVIDER_coreutils = "uutils-coreutils"` in your `build/conf/local.conf` file and +then build your usual yocto image. + ## Non-standard packages -### `coreutils-hybrid` (AUR) +### `coreutils-uutils` (AUR) -[![AUR package](https://repology.org/badge/version-for-repo/aur/coreutils-hybrid.svg)](https://aur.archlinux.org/packages/coreutils-hybrid) +[AUR package](https://aur.archlinux.org/packages/coreutils-uutils) -A GNU coreutils / uutils coreutils hybrid package. Uses stable uutils -programs mixed with GNU counterparts if uutils counterpart is -unfinished or buggy. +Cross-platform Rust rewrite of the GNU coreutils being used as actual system coreutils. diff --git a/docs/src/performance.md b/docs/src/performance.md new file mode 100644 index 00000000000..39dd6a969e8 --- /dev/null +++ b/docs/src/performance.md @@ -0,0 +1,108 @@ + + +# Performance Profiling Tutorial + +## Effective Benchmarking with Hyperfine + +[Hyperfine](https://github.com/sharkdp/hyperfine) is a powerful command-line benchmarking tool that allows you to measure and compare execution times of commands with statistical rigor. + +### Benchmarking Best Practices + +When evaluating performance improvements, always set up your benchmarks to compare: + +1. The GNU implementation as reference +2. The implementation without the change +3. The implementation with your change + +This three-way comparison provides clear insights into: +- How your implementation compares to the standard (GNU) +- The actual performance impact of your specific change + +### Example Benchmark + +First, you will need to build the binary in release mode. Debug builds are significantly slower: + +```bash +cargo build --features unix --release +``` + +```bash +# Three-way comparison benchmark +hyperfine \ + --warmup 3 \ + "/usr/bin/ls -R ." \ + "./target/release/coreutils.prev ls -R ." \ + "./target/release/coreutils ls -R ." + +# can be simplified with: +hyperfine \ + --warmup 3 \ + -L ls /usr/bin/ls,"./target/release/coreutils.prev ls","./target/release/coreutils ls" \ + "{ls} -R ." +``` + +``` +# to improve the reproducibility of the results: +taskset -c 0 +``` + +### Interpreting Results + +Hyperfine provides summary statistics including: +- Mean execution time +- Standard deviation +- Min/max times +- Relative performance comparison + +Look for consistent patterns rather than focusing on individual runs, and be aware of system noise that might affect results. + +## Using Samply for Profiling + +[Samply](https://github.com/mstange/samply) is a sampling profiler that helps you identify performance bottlenecks in your code. + +### Basic Profiling + +```bash +# Generate a flame graph for your application +samply record ./target/debug/coreutils ls -R + +# Profile with higher sampling frequency +samply record --rate 1000 ./target/debug/coreutils seq 1 1000 +``` + +The output using the `debug` profile might be easier to understand, but the performance characteristics may be somewhat different from `release` profile that we _actually_ care about. + +Consider using the `profiling` profile, that compiles in `release` mode but with debug symbols. For example: +```bash +cargo build --profile profiling -p uu_ls +samply record -r 10000 target/profiling/ls -lR /var .git .git .git > /dev/null +``` + +## Workflow: Measuring Performance Improvements + +1. **Establish baselines**: + ```bash + hyperfine --warmup 3 \ + "/usr/bin/sort large_file.txt" \ + "our-sort-v1 large_file.txt" + ``` + +2. **Identify bottlenecks**: + ```bash + samply record ./our-sort-v1 large_file.txt + ``` + +3. **Make targeted improvements** based on profiling data + +4. **Verify improvements**: + ```bash + hyperfine --warmup 3 \ + "/usr/bin/sort large_file.txt" \ + "our-sort-v1 large_file.txt" \ + "our-sort-v2 large_file.txt" + ``` + +5. **Document performance changes** with concrete numbers + ```bash + hyperfine --export-markdown file.md [...] + ``` diff --git a/docs/src/test_coverage.js b/docs/src/test_coverage.js index e601229affc..318c9934d53 100644 --- a/docs/src/test_coverage.js +++ b/docs/src/test_coverage.js @@ -19,7 +19,7 @@ function progressBar(totals) { var(--SKIP) ${skipPercentage}%` ) + (skipPercentage === 100 ? ")" : ", var(--FAIL) 0)"); - + const progress = document.createElement("div"); progress.className = "progress" progress.innerHTML = ` @@ -74,7 +74,7 @@ function parse_result(parent, obj) { return totals; } -fetch("https://raw.githubusercontent.com/uutils/coreutils-tracking/main/gnu-full-result.json") +fetch("https://raw.githubusercontent.com/uutils/coreutils-tracking/main/aggregated-result.json") .then((r) => r.json()) .then((obj) => { let parent = document.getElementById("test-cov"); diff --git a/docs/src/test_coverage.md b/docs/src/test_coverage.md index b8376058873..2bfad68bcac 100644 --- a/docs/src/test_coverage.md +++ b/docs/src/test_coverage.md @@ -18,4 +18,4 @@ or resulted in an error. ## Progress over time - + diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000000..fbf85d3df31 --- /dev/null +++ b/flake.lock @@ -0,0 +1,42 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1720633750, + "narHash": "sha256-N8apMO2pP/upWeH+JY5eM8VDp2qBAAzE+OY5LRW6qpw=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "54bc082f5a7219d122e74fe52c021cf59fed9d6f", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "systems": "systems" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000000..79c69c4901e --- /dev/null +++ b/flake.nix @@ -0,0 +1,74 @@ +# spell-checker:ignore bintools gnum gperf ldflags libclang nixpkgs numtide pkgs texinfo +{ + inputs = { + nixpkgs.url = "github:nixos/nixpkgs"; + + # + systems.url = "github:nix-systems/default"; + }; + + outputs = inputs: let + inherit (inputs.nixpkgs) lib legacyPackages; + eachSystem = lib.genAttrs (import inputs.systems); + pkgsFor = legacyPackages; + in { + devShells = eachSystem ( + system: let + libselinuxPath = with pkgsFor.${system}; + lib.makeLibraryPath [ + libselinux + ]; + + libaclPath = with pkgsFor.${system}; + lib.makeLibraryPath [ + acl + ]; + + build_deps = with pkgsFor.${system}; [ + clang + llvmPackages.bintools + rustup + + pre-commit + + # debugging + gdb + ]; + + gnu_testing_deps = with pkgsFor.${system}; [ + autoconf + automake + bison + gnum4 + gperf + gettext + texinfo + ]; + in { + default = pkgsFor.${system}.pkgs.mkShell { + packages = build_deps ++ gnu_testing_deps; + + RUSTC_VERSION = "1.85"; + LIBCLANG_PATH = pkgsFor.${system}.lib.makeLibraryPath [pkgsFor.${system}.llvmPackages_latest.libclang.lib]; + shellHook = '' + export PATH=$PATH:''${CARGO_HOME:-~/.cargo}/bin + export PATH=$PATH:''${RUSTUP_HOME:-~/.rustup}/toolchains/$RUSTC_VERSION-x86_64-unknown-linux-gnu/bin/ + ''; + + SELINUX_INCLUDE_DIR = ''${pkgsFor.${system}.libselinux.dev}/include''; + SELINUX_LIB_DIR = libselinuxPath; + SELINUX_STATIC = "0"; + + # Necessary to build GNU. + LDFLAGS = ''-L ${libselinuxPath} -L ${libaclPath}''; + + # Add precompiled library to rustc search path + RUSTFLAGS = [ + ''-L ${libselinuxPath}'' + ''-L ${libaclPath}'' + ]; + }; + } + ); + }; +} diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock new file mode 100644 index 00000000000..37b66a3fa31 --- /dev/null +++ b/fuzz/Cargo.lock @@ -0,0 +1,1647 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys", +] + +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bigdecimal" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "binary-heap-plus" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4551d8382e911ecc0d0f0ffb602777988669be09447d536ff4388d1def11296" +dependencies = [ + "compare", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "blake2b_simd" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06e903a20b159e944f91ec8499fe1e55651480c541ea0a584f5d967c49ad9d99" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "blake3" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + +[[package]] +name = "cc" +version = "1.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efdce149c370f133a071ca8ef6ea340b7b88748ab0810097a9e2976eaa34b4f3" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f10f8c9340e31fc120ff885fcdb54a0b48e474bbd77cab557f0c30a3e569402" +dependencies = [ + "parse-zoneinfo", + "phf_codegen", +] + +[[package]] +name = "clap" +version = "4.5.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "compare" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120133d4db2ec47efe2e26502ee984747630c67f51974fca0b6c1340cf2368d3" + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697b5419f348fd5ae2478e8018cb016c00a5881c7f46c717de98ffd135a5651c" +dependencies = [ + "nix", + "windows-sys", +] + +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + +[[package]] +name = "data-encoding-macro" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47ce6c96ea0102f01122a185683611bd5ac8d99e62bc59dd12e6bda344ee673d" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "errno" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.2", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libm" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9627da5196e5d8ed0b0495e61e518847578da83483c37288316d9b2e03a7f72" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags 1.3.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown", +] + +[[package]] +name = "os_display" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad5fd71b79026fb918650dde6d125000a233764f1c2f1659a1c71118e33ea08f" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + +[[package]] +name = "parse_datetime" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd3830b49ee3a0dcc8fdfadc68c6354c97d00101ac1cac5b2eee25d35c42066" +dependencies = [ + "chrono", + "nom", + "regex", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rust-ini" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" +dependencies = [ + "cfg-if", + "ordered-multimap", + "trim-in-place", +] + +[[package]] +name = "rustix" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +dependencies = [ + "bitflags 2.9.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "self_cell" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + +[[package]] +name = "sm3" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb9a3b702d0a7e33bc4d85a14456633d2b165c2ad839c5fd9a8417c1ab15860" +dependencies = [ + "digest", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "terminal_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" +dependencies = [ + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "trim-in-place" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uu_cksum" +version = "0.0.30" +dependencies = [ + "clap", + "hex", + "regex", + "uucore", +] + +[[package]] +name = "uu_cut" +version = "0.0.30" +dependencies = [ + "bstr", + "clap", + "memchr", + "uucore", +] + +[[package]] +name = "uu_date" +version = "0.0.30" +dependencies = [ + "chrono", + "clap", + "libc", + "parse_datetime", + "uucore", + "windows-sys", +] + +[[package]] +name = "uu_echo" +version = "0.0.30" +dependencies = [ + "clap", + "uucore", +] + +[[package]] +name = "uu_env" +version = "0.0.30" +dependencies = [ + "clap", + "nix", + "rust-ini", + "thiserror", + "uucore", +] + +[[package]] +name = "uu_expr" +version = "0.0.30" +dependencies = [ + "clap", + "num-bigint", + "num-traits", + "onig", + "thiserror", + "uucore", +] + +[[package]] +name = "uu_printf" +version = "0.0.30" +dependencies = [ + "clap", + "uucore", +] + +[[package]] +name = "uu_seq" +version = "0.0.30" +dependencies = [ + "bigdecimal", + "clap", + "num-bigint", + "num-traits", + "thiserror", + "uucore", +] + +[[package]] +name = "uu_sort" +version = "0.0.30" +dependencies = [ + "binary-heap-plus", + "clap", + "compare", + "ctrlc", + "fnv", + "itertools", + "memchr", + "nix", + "rand 0.9.1", + "rayon", + "self_cell", + "tempfile", + "thiserror", + "unicode-width", + "uucore", +] + +[[package]] +name = "uu_split" +version = "0.0.30" +dependencies = [ + "clap", + "memchr", + "thiserror", + "uucore", +] + +[[package]] +name = "uu_test" +version = "0.0.30" +dependencies = [ + "clap", + "libc", + "uucore", +] + +[[package]] +name = "uu_tr" +version = "0.0.30" +dependencies = [ + "clap", + "nom", + "uucore", +] + +[[package]] +name = "uu_wc" +version = "0.0.30" +dependencies = [ + "bytecount", + "clap", + "libc", + "nix", + "thiserror", + "unicode-width", + "uucore", +] + +[[package]] +name = "uucore" +version = "0.0.30" +dependencies = [ + "bigdecimal", + "blake2b_simd", + "blake3", + "chrono", + "chrono-tz", + "clap", + "crc32fast", + "data-encoding", + "data-encoding-macro", + "digest", + "dunce", + "glob", + "hex", + "iana-time-zone", + "itertools", + "libc", + "md-5", + "memchr", + "nix", + "num-traits", + "number_prefix", + "os_display", + "sha1", + "sha2", + "sha3", + "sm3", + "thiserror", + "uucore_procs", + "wild", + "winapi-util", + "windows-sys", + "z85", +] + +[[package]] +name = "uucore-fuzz" +version = "0.0.0" +dependencies = [ + "console", + "libc", + "libfuzzer-sys", + "rand 0.9.1", + "similar", + "tempfile", + "uu_cksum", + "uu_cut", + "uu_date", + "uu_echo", + "uu_env", + "uu_expr", + "uu_printf", + "uu_seq", + "uu_sort", + "uu_split", + "uu_test", + "uu_tr", + "uu_wc", + "uucore", +] + +[[package]] +name = "uucore_procs" +version = "0.0.30" +dependencies = [ + "proc-macro2", + "quote", + "uuhelp_parser", +] + +[[package]] +name = "uuhelp_parser" +version = "0.0.30" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wild" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3131afc8c575281e1e80f36ed6a092aa502c08b18ed7524e86fbbb12bb410e1" +dependencies = [ + "glob", +] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.9.0", +] + +[[package]] +name = "z85" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3a41ce106832b4da1c065baa4c31cf640cf965fa1483816402b7f6b96f0a64" + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index dfb62aba5e1..255d11d5b72 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -2,19 +2,20 @@ name = "uucore-fuzz" version = "0.0.0" publish = false -edition = "2021" +edition = "2024" [package.metadata] cargo-fuzz = true [dependencies] -libfuzzer-sys = "0.4" -libc = "0.2" -tempfile = "3" -rand = { version = "0.8", features = ["small_rng"] } -similar = "2" - -uucore = { path = "../src/uucore/" } +console = "0.15.0" +libfuzzer-sys = "0.4.7" +libc = "0.2.153" +tempfile = "3.15.0" +rand = { version = "0.9.0", features = ["small_rng"] } +similar = "2.5.0" + +uucore = { path = "../src/uucore/", features = ["parser"] } uu_date = { path = "../src/uu/date/" } uu_test = { path = "../src/uu/test/" } uu_expr = { path = "../src/uu/expr/" } @@ -25,6 +26,9 @@ uu_sort = { path = "../src/uu/sort/" } uu_wc = { path = "../src/uu/wc/" } uu_cut = { path = "../src/uu/cut/" } uu_split = { path = "../src/uu/split/" } +uu_tr = { path = "../src/uu/tr/" } +uu_env = { path = "../src/uu/env/" } +uu_cksum = { path = "../src/uu/cksum/" } # Prevent this from interfering with workspaces [workspace] @@ -90,6 +94,12 @@ path = "fuzz_targets/fuzz_test.rs" test = false doc = false +[[bin]] +name = "fuzz_seq_parse_number" +path = "fuzz_targets/fuzz_seq_parse_number.rs" +test = false +doc = false + [[bin]] name = "fuzz_parse_glob" path = "fuzz_targets/fuzz_parse_glob.rs" @@ -107,3 +117,21 @@ name = "fuzz_parse_time" path = "fuzz_targets/fuzz_parse_time.rs" test = false doc = false + +[[bin]] +name = "fuzz_tr" +path = "fuzz_targets/fuzz_tr.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_env" +path = "fuzz_targets/fuzz_env.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_cksum" +path = "fuzz_targets/fuzz_cksum.rs" +test = false +doc = false diff --git a/fuzz/fuzz_targets/fuzz_cksum.rs b/fuzz/fuzz_targets/fuzz_cksum.rs new file mode 100644 index 00000000000..3b5ddb8bb18 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_cksum.rs @@ -0,0 +1,171 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore chdir + +#![no_main] +use libfuzzer_sys::fuzz_target; +use std::ffi::OsString; +use uu_cksum::uumain; +mod fuzz_common; +use crate::fuzz_common::{ + CommandResult, compare_result, generate_and_run_uumain, generate_random_file, + generate_random_string, + pretty_print::{print_or_empty, print_test_begin}, + replace_fuzz_binary_name, run_gnu_cmd, +}; +use rand::Rng; +use std::env::temp_dir; +use std::fs::{self, File}; +use std::io::Write; +use std::process::Command; + +static CMD_PATH: &str = "cksum"; + +fn generate_cksum_args() -> Vec { + let mut rng = rand::rng(); + let mut args = Vec::new(); + + let digests = [ + "sysv", "bsd", "crc", "md5", "sha1", "sha224", "sha256", "sha384", "sha512", "blake2b", + "sm3", + ]; + let digest_opts = [ + "--base64", + "--raw", + "--tag", + "--untagged", + "--text", + "--binary", + ]; + + if rng.random_bool(0.3) { + args.push("-a".to_string()); + args.push(digests[rng.random_range(0..digests.len())].to_string()); + } + + if rng.random_bool(0.2) { + args.push(digest_opts[rng.random_range(0..digest_opts.len())].to_string()); + } + + if rng.random_bool(0.15) { + args.push("-l".to_string()); + args.push(rng.random_range(8..513).to_string()); + } + + if rng.random_bool(0.05) { + for _ in 0..rng.random_range(0..3) { + args.push(format!("file_{}", generate_random_string(5))); + } + } else { + args.push("-c".to_string()); + } + + if rng.random_bool(0.25) { + if let Ok(file_path) = generate_random_file() { + args.push(file_path); + } + } + + if args.is_empty() || !args.iter().any(|arg| arg.starts_with("file_")) { + args.push("-a".to_string()); + args.push(digests[rng.random_range(0..digests.len())].to_string()); + + if let Ok(file_path) = generate_random_file() { + args.push(file_path); + } + } + + args +} + +fn generate_checksum_file( + algo: &str, + file_path: &str, + digest_opts: &[&str], +) -> Result { + let checksum_file_path = temp_dir().join("checksum_file"); + let mut cmd = Command::new(CMD_PATH); + cmd.arg("-a").arg(algo); + + for opt in digest_opts { + cmd.arg(opt); + } + + cmd.arg(file_path); + let output = cmd.output()?; + + let mut checksum_file = File::create(&checksum_file_path)?; + checksum_file.write_all(&output.stdout)?; + + Ok(checksum_file_path.to_str().unwrap().to_string()) +} + +fn select_random_digest_opts<'a>( + rng: &mut rand::rngs::ThreadRng, + digest_opts: &'a [&'a str], +) -> Vec<&'a str> { + digest_opts + .iter() + .filter(|_| rng.random_bool(0.5)) + .copied() + .collect() +} + +fuzz_target!(|_data: &[u8]| { + let cksum_args = generate_cksum_args(); + let mut args = vec![OsString::from("cksum")]; + args.extend(cksum_args.iter().map(OsString::from)); + + if let Ok(file_path) = generate_random_file() { + let algo = cksum_args + .iter() + .position(|arg| arg == "-a") + .map_or("md5", |index| &cksum_args[index + 1]); + + let all_digest_opts = ["--base64", "--raw", "--tag", "--untagged"]; + let mut rng = rand::rng(); + let selected_digest_opts = select_random_digest_opts(&mut rng, &all_digest_opts); + + if let Ok(checksum_file_path) = + generate_checksum_file(algo, &file_path, &selected_digest_opts) + { + print_test_begin(format!("cksum {args:?}")); + + if let Ok(content) = fs::read_to_string(&checksum_file_path) { + println!("File content ({checksum_file_path})"); + print_or_empty(&content); + } else { + eprintln!("Error reading the checksum file."); + } + let mut rust_result = generate_and_run_uumain(&args, uumain, None); + + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + // Lower the number of false positives caused by binary names + replace_fuzz_binary_name("cksum", &mut rust_result); + + compare_result( + "cksum", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, + ); + } + } +}); diff --git a/fuzz/fuzz_targets/fuzz_common.rs b/fuzz/fuzz_targets/fuzz_common/mod.rs similarity index 78% rename from fuzz/fuzz_targets/fuzz_common.rs rename to fuzz/fuzz_targets/fuzz_common/mod.rs index cf56268d75a..e887bfc6755 100644 --- a/fuzz/fuzz_targets/fuzz_common.rs +++ b/fuzz/fuzz_targets/fuzz_common/mod.rs @@ -3,19 +3,26 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +use console::Style; use libc::STDIN_FILENO; -use libc::{close, dup, dup2, pipe, STDERR_FILENO, STDOUT_FILENO}; -use rand::prelude::SliceRandom; +use libc::{STDERR_FILENO, STDOUT_FILENO, close, dup, dup2, pipe}; +use pretty_print::{ + print_diff, print_end_with_status, print_or_empty, print_section, print_with_style, +}; use rand::Rng; -use similar::TextDiff; +use rand::prelude::IndexedRandom; +use std::env::temp_dir; use std::ffi::OsString; +use std::fs::File; use std::io::{Seek, SeekFrom, Write}; use std::os::fd::{AsRawFd, RawFd}; use std::process::{Command, Stdio}; use std::sync::atomic::Ordering; -use std::sync::{atomic::AtomicBool, Once}; +use std::sync::{Once, atomic::AtomicBool}; use std::{io, thread}; +pub mod pretty_print; + /// Represents the result of running a command, including its standard output, /// standard error, and exit code. pub struct CommandResult { @@ -36,7 +43,7 @@ pub fn is_gnu_cmd(cmd_path: &str) -> Result<(), std::io::Error> { CHECK_GNU.call_once(|| { let version_output = Command::new(cmd_path).arg("--version").output().unwrap(); - println!("version_output {:#?}", version_output); + println!("version_output {version_output:#?}"); let version_str = String::from_utf8_lossy(&version_output.stdout).to_string(); if version_str.contains("GNU coreutils") { @@ -105,7 +112,7 @@ where let original_stdin_fd = if let Some(input_str) = pipe_input { // we have pipe input let mut input_file = tempfile::tempfile().unwrap(); - write!(input_file, "{}", input_str).unwrap(); + write!(input_file, "{input_str}").unwrap(); input_file.seek(SeekFrom::Start(0)).unwrap(); // Redirect stdin to read from the in-memory file @@ -125,6 +132,8 @@ where let (uumain_exit_status, captured_stdout, captured_stderr) = thread::scope(|s| { let out = s.spawn(|| read_from_fd(pipe_stdout_fds[0])); let err = s.spawn(|| read_from_fd(pipe_stderr_fds[0])); + #[allow(clippy::unnecessary_to_owned)] + // TODO: clippy wants us to use args.iter().cloned() ? let status = uumain_function(args.to_owned().into_iter()); // Reset the exit code global variable in case we run another test after this one // See https://github.com/uutils/coreutils/issues/5777 @@ -231,6 +240,10 @@ pub fn run_gnu_cmd( command.arg(arg); } + // See https://github.com/uutils/coreutils/issues/6794 + // uutils' coreutils is not locale-aware, and aims to mirror/be compatible with GNU Core Utilities's LC_ALL=C behavior + command.env("LC_ALL", "C"); + let output = if let Some(input_str) = pipe_input { // We have an pipe input command @@ -309,10 +322,10 @@ pub fn compare_result( gnu_result: &CommandResult, fail_on_stderr_diff: bool, ) { - println!("Test Type: {}", test_type); - println!("Input: {}", input); + print_section(format!("Compare result for: {test_type} {input}")); + if let Some(pipe) = pipe_input { - println!("Pipe: {}", pipe); + println!("Pipe: {pipe}"); } let mut discrepancies = Vec::new(); @@ -320,62 +333,68 @@ pub fn compare_result( if rust_result.stdout.trim() != gnu_result.stdout.trim() { discrepancies.push("stdout differs"); - println!("Rust stdout: {}", rust_result.stdout); - println!("GNU stdout: {}", gnu_result.stdout); + println!("Rust stdout:"); + print_or_empty(rust_result.stdout.as_str()); + println!("GNU stdout:"); + print_or_empty(gnu_result.stdout.as_ref()); print_diff(&rust_result.stdout, &gnu_result.stdout); should_panic = true; } + if rust_result.stderr.trim() != gnu_result.stderr.trim() { discrepancies.push("stderr differs"); - println!("Rust stderr: {}", rust_result.stderr); - println!("GNU stderr: {}", gnu_result.stderr); + println!("Rust stderr:"); + print_or_empty(rust_result.stderr.as_str()); + println!("GNU stderr:"); + print_or_empty(gnu_result.stderr.as_str()); print_diff(&rust_result.stderr, &gnu_result.stderr); if fail_on_stderr_diff { should_panic = true; } } + if rust_result.exit_code != gnu_result.exit_code { discrepancies.push("exit code differs"); - println!("Rust exit code: {}", rust_result.exit_code); - println!("GNU exit code: {}", gnu_result.exit_code); + println!( + "Different exit code: (Rust: {}, GNU: {})", + rust_result.exit_code, gnu_result.exit_code + ); should_panic = true; } if discrepancies.is_empty() { - println!("All outputs and exit codes matched."); + print_end_with_status("Same behavior", true); } else { - println!("Discrepancy detected: {}", discrepancies.join(", ")); + print_with_style( + format!("Discrepancies detected: {}", discrepancies.join(", ")), + Style::new().red(), + ); if should_panic { - panic!("Test failed for {}: {}", test_type, input); + print_end_with_status( + format!("Test failed and will panic for: {test_type} {input}"), + false, + ); + panic!("Test failed for: {test_type} {input}"); } else { - println!( - "Test completed with discrepancies for {}: {}", - test_type, input + print_end_with_status( + format!("Test completed with discrepancies for: {test_type} {input}"), + false, ); } } -} - -/// When we have different outputs, print the diff -fn print_diff(rust_output: &str, gnu_output: &str) { - println!("Diff="); - let diff = TextDiff::from_lines(rust_output, gnu_output); - for change in diff.iter_all_changes() { - print!("{}{}", change.tag(), change); - } println!(); } pub fn generate_random_string(max_length: usize) -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let valid_utf8: Vec = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" .chars() .collect(); let invalid_utf8 = [0xC3, 0x28]; // Invalid UTF-8 sequence let mut result = String::new(); - for _ in 0..rng.gen_range(1..=max_length) { - if rng.gen_bool(0.9) { + for _ in 0..rng.random_range(0..=max_length) { + if rng.random_bool(0.9) { let ch = valid_utf8.choose(&mut rng).unwrap(); result.push(*ch); } else { @@ -388,3 +407,32 @@ pub fn generate_random_string(max_length: usize) -> String { result } + +#[allow(dead_code)] +pub fn generate_random_file() -> Result { + let mut rng = rand::rng(); + let file_name: String = (0..10) + .map(|_| rng.random_range(b'a'..=b'z') as char) + .collect(); + let mut file_path = temp_dir(); + file_path.push(file_name); + + let mut file = File::create(&file_path)?; + + let content_length = rng.random_range(10..1000); + let content: String = (0..content_length) + .map(|_| (rng.random_range(b' '..=b'~') as char)) + .collect(); + + file.write_all(content.as_bytes())?; + + Ok(file_path.to_str().unwrap().to_string()) +} + +#[allow(dead_code)] +pub fn replace_fuzz_binary_name(cmd: &str, result: &mut CommandResult) { + let fuzz_bin_name = format!("fuzz/target/x86_64-unknown-linux-gnu/release/fuzz_{cmd}"); + + result.stdout = result.stdout.replace(&fuzz_bin_name, cmd); + result.stderr = result.stderr.replace(&fuzz_bin_name, cmd); +} diff --git a/fuzz/fuzz_targets/fuzz_common/pretty_print.rs b/fuzz/fuzz_targets/fuzz_common/pretty_print.rs new file mode 100644 index 00000000000..ecdfccfd035 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_common/pretty_print.rs @@ -0,0 +1,69 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::fmt; + +use console::{Style, style}; +use similar::TextDiff; + +pub fn print_section(s: S) { + println!("{}", style(format!("=== {s}")).bold()); +} + +pub fn print_subsection(s: S) { + println!("{}", style(format!("--- {s}")).bright()); +} + +#[allow(dead_code)] +pub fn print_test_begin(msg: S) { + println!( + "{} {} {}", + style("===").bold(), // Kind of gray + style("TEST").black().on_yellow().bold(), + style(msg).bold() + ); +} + +pub fn print_end_with_status(msg: S, ok: bool) { + let ok = if ok { + style(" OK ").black().on_green().bold() + } else { + style(" KO ").black().on_red().bold() + }; + + println!( + "{} {ok} {}", + style("===").bold(), // Kind of gray + style(msg).bold() + ); +} + +pub fn print_or_empty(s: &str) { + let to_print = if s.is_empty() { "(empty)" } else { s }; + + println!("{}", style(to_print).dim()); +} + +pub fn print_with_style(msg: S, style: Style) { + println!("{}", style.apply_to(msg)); +} + +pub fn print_diff(got: &str, expected: &str) { + let diff = TextDiff::from_lines(got, expected); + + print_subsection("START diff"); + + for change in diff.iter_all_changes() { + let (sign, style) = match change.tag() { + similar::ChangeTag::Equal => (" ", Style::new().dim()), + similar::ChangeTag::Delete => ("-", Style::new().red()), + similar::ChangeTag::Insert => ("+", Style::new().green()), + }; + print!("{}{}", style.apply_to(sign).bold(), style.apply_to(change)); + } + + print_subsection("END diff"); + println!(); +} diff --git a/fuzz/fuzz_targets/fuzz_cut.rs b/fuzz/fuzz_targets/fuzz_cut.rs index fa5f8fcc472..828a7c6190b 100644 --- a/fuzz/fuzz_targets/fuzz_cut.rs +++ b/fuzz/fuzz_targets/fuzz_cut.rs @@ -13,24 +13,24 @@ use std::ffi::OsString; mod fuzz_common; use crate::fuzz_common::{ - compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, CommandResult, + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, }; static CMD_PATH: &str = "cut"; fn generate_cut_args() -> String { - let mut rng = rand::thread_rng(); - let arg_count = rng.gen_range(1..=6); + let mut rng = rand::rng(); + let arg_count = rng.random_range(1..=6); let mut args = Vec::new(); for _ in 0..arg_count { - if rng.gen_bool(0.1) { - args.push(generate_random_string(rng.gen_range(1..=20))); + if rng.random_bool(0.1) { + args.push(generate_random_string(rng.random_range(1..=20))); } else { - match rng.gen_range(0..=4) { - 0 => args.push(String::from("-b") + &rng.gen_range(1..=10).to_string()), - 1 => args.push(String::from("-c") + &rng.gen_range(1..=10).to_string()), + match rng.random_range(0..=4) { + 0 => args.push(String::from("-b") + &rng.random_range(1..=10).to_string()), + 1 => args.push(String::from("-c") + &rng.random_range(1..=10).to_string()), 2 => args.push(String::from("-d,") + &generate_random_string(1)), // Using a comma as a default delimiter - 3 => args.push(String::from("-f") + &rng.gen_range(1..=5).to_string()), + 3 => args.push(String::from("-f") + &rng.random_range(1..=5).to_string()), _ => (), } } @@ -40,12 +40,12 @@ fn generate_cut_args() -> String { } fn generate_delimited_data(count: usize) -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let mut lines = Vec::new(); for _ in 0..count { - let fields = (0..rng.gen_range(1..=5)) - .map(|_| generate_random_string(rng.gen_range(1..=10))) + let fields = (0..rng.random_range(1..=5)) + .map(|_| generate_random_string(rng.random_range(1..=10))) .collect::>() .join(","); lines.push(fields); diff --git a/fuzz/fuzz_targets/fuzz_echo.rs b/fuzz/fuzz_targets/fuzz_echo.rs index 3f15b257e6e..a36a7ebadfc 100644 --- a/fuzz/fuzz_targets/fuzz_echo.rs +++ b/fuzz/fuzz_targets/fuzz_echo.rs @@ -2,8 +2,8 @@ use libfuzzer_sys::fuzz_target; use uu_echo::uumain; -use rand::prelude::SliceRandom; use rand::Rng; +use rand::prelude::IndexedRandom; use std::ffi::OsString; mod fuzz_common; @@ -15,13 +15,14 @@ use crate::fuzz_common::{ static CMD_PATH: &str = "echo"; fn generate_echo() -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let mut echo_str = String::new(); // Randomly decide whether to include options - let include_n = rng.gen_bool(0.1); // 10% chance - let include_e = rng.gen_bool(0.1); // 10% chance - let include_E = rng.gen_bool(0.1); // 10% chance + let include_n = rng.random_bool(0.1); // 10% chance + let include_e = rng.random_bool(0.1); // 10% chance + #[allow(non_snake_case)] + let include_E = rng.random_bool(0.1); // 10% chance if include_n { echo_str.push_str("-n "); @@ -34,12 +35,12 @@ fn generate_echo() -> String { } // Add a random string - echo_str.push_str(&generate_random_string(rng.gen_range(1..=10))); + echo_str.push_str(&generate_random_string(rng.random_range(1..=10))); // Include escape sequences if -e is enabled if include_e { // Add a 10% chance of including an escape sequence - if rng.gen_bool(0.1) { + if rng.random_bool(0.1) { echo_str.push_str(&generate_escape_sequence(&mut rng)); } } diff --git a/fuzz/fuzz_targets/fuzz_env.rs b/fuzz/fuzz_targets/fuzz_env.rs new file mode 100644 index 00000000000..f38dced076e --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_env.rs @@ -0,0 +1,97 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore chdir + +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_env::uumain; + +use std::ffi::OsString; + +mod fuzz_common; +use crate::fuzz_common::{ + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, +}; +use rand::Rng; + +static CMD_PATH: &str = "env"; + +fn generate_env_args() -> Vec { + let mut rng = rand::rng(); + let mut args = Vec::new(); + + let opts = ["-i", "-0", "-v", "-vv"]; + for opt in &opts { + if rng.random_bool(0.2) { + args.push(opt.to_string()); + } + } + + if rng.random_bool(0.3) { + args.push(format!( + "-u={}", + generate_random_string(rng.random_range(3..10)) + )); + } + + if rng.random_bool(0.2) { + args.push(format!("--chdir={}", "/tmp")); // Simplified example + } + + /* + Options not implemented for now + if rng.random_bool(0.15) { + let sig_opts = ["--block-signal"];//, /*"--default-signal",*/ "--ignore-signal"]; + let chosen_sig_opt = sig_opts[rng.random_range(0..sig_opts.len())]; + args.push(chosen_sig_opt.to_string()); + // Simplify by assuming SIGPIPE for demonstration + if !chosen_sig_opt.ends_with("list-signal-handling") { + args.push(String::from("SIGPIPE")); + } + }*/ + + // Adding a few random NAME=VALUE pairs + for _ in 0..rng.random_range(0..3) { + args.push(format!( + "{}={}", + generate_random_string(5), + generate_random_string(5) + )); + } + + args +} + +fuzz_target!(|_data: &[u8]| { + let env_args = generate_env_args(); + let mut args = vec![OsString::from("env")]; + args.extend(env_args.iter().map(OsString::from)); + let input_lines = generate_random_string(10); + + let rust_result = generate_and_run_uumain(&args, uumain, Some(&input_lines)); + + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "env", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_expr.rs b/fuzz/fuzz_targets/fuzz_expr.rs index 8bc18fae47e..a2c232ab333 100644 --- a/fuzz/fuzz_targets/fuzz_expr.rs +++ b/fuzz/fuzz_targets/fuzz_expr.rs @@ -8,8 +8,8 @@ use libfuzzer_sys::fuzz_target; use uu_expr::uumain; -use rand::seq::SliceRandom; use rand::Rng; +use rand::prelude::IndexedRandom; use std::{env, ffi::OsString}; mod fuzz_common; @@ -20,9 +20,10 @@ use crate::fuzz_common::{ static CMD_PATH: &str = "expr"; fn generate_expr(max_depth: u32) -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let ops = [ - "+", "-", "*", "/", "%", "<", ">", "=", "&", "|", "!=", "<=", ">=", ":", "index", "length", "substr", + "+", "-", "*", "/", "%", "<", ">", "=", "&", "|", "!=", "<=", ">=", ":", "index", "length", + "substr", ]; let mut expr = String::new(); @@ -32,18 +33,18 @@ fn generate_expr(max_depth: u32) -> String { while depth <= max_depth { if last_was_operator || depth == 0 { // Add a number - expr.push_str(&rng.gen_range(1..=100).to_string()); + expr.push_str(&rng.random_range(1..=100).to_string()); last_was_operator = false; } else { // 90% chance to add an operator followed by a number - if rng.gen_bool(0.9) { + if rng.random_bool(0.9) { let op = *ops.choose(&mut rng).unwrap(); - expr.push_str(&format!(" {} ", op)); + expr.push_str(&format!(" {op} ")); last_was_operator = true; } // 10% chance to add a random string (potentially invalid syntax) else { - let random_str = generate_random_string(rng.gen_range(1..=10)); + let random_str = generate_random_string(rng.random_range(1..=10)); expr.push_str(&random_str); last_was_operator = false; } @@ -53,22 +54,24 @@ fn generate_expr(max_depth: u32) -> String { // Ensure the expression ends with a number if it ended with an operator if last_was_operator { - expr.push_str(&rng.gen_range(1..=100).to_string()); + expr.push_str(&rng.random_range(1..=100).to_string()); } expr } fuzz_target!(|_data: &[u8]| { - let mut rng = rand::thread_rng(); - let expr = generate_expr(rng.gen_range(0..=20)); + let mut rng = rand::rng(); + let expr = generate_expr(rng.random_range(0..=20)); let mut args = vec![OsString::from("expr")]; args.extend(expr.split_whitespace().map(OsString::from)); // Use C locale to avoid false positives, like in https://github.com/uutils/coreutils/issues/5378, // because uutils expr doesn't support localization yet // TODO remove once uutils expr supports localization - env::set_var("LC_COLLATE", "C"); + unsafe { + env::set_var("LC_COLLATE", "C"); + } let rust_result = generate_and_run_uumain(&args, uumain, None); let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { diff --git a/fuzz/fuzz_targets/fuzz_parse_glob.rs b/fuzz/fuzz_targets/fuzz_parse_glob.rs index e235c0c9d89..66e772959e7 100644 --- a/fuzz/fuzz_targets/fuzz_parse_glob.rs +++ b/fuzz/fuzz_targets/fuzz_parse_glob.rs @@ -1,7 +1,7 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use uucore::parse_glob; +use uucore::parser::parse_glob; fuzz_target!(|data: &[u8]| { if let Ok(s) = std::str::from_utf8(data) { diff --git a/fuzz/fuzz_targets/fuzz_parse_size.rs b/fuzz/fuzz_targets/fuzz_parse_size.rs index d032adf0666..4e8d7e2216b 100644 --- a/fuzz/fuzz_targets/fuzz_parse_size.rs +++ b/fuzz/fuzz_targets/fuzz_parse_size.rs @@ -1,7 +1,7 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use uucore::parse_size::parse_size_u64; +use uucore::parser::parse_size::parse_size_u64; fuzz_target!(|data: &[u8]| { if let Ok(s) = std::str::from_utf8(data) { diff --git a/fuzz/fuzz_targets/fuzz_parse_time.rs b/fuzz/fuzz_targets/fuzz_parse_time.rs index a643c6d805c..5745e5c8709 100644 --- a/fuzz/fuzz_targets/fuzz_parse_time.rs +++ b/fuzz/fuzz_targets/fuzz_parse_time.rs @@ -1,10 +1,11 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use uucore::parse_time; +use uucore::parser::parse_time; fuzz_target!(|data: &[u8]| { if let Ok(s) = std::str::from_utf8(data) { - _ = parse_time::from_str(s); + _ = parse_time::from_str(s, true); + _ = parse_time::from_str(s, false); } }); diff --git a/fuzz/fuzz_targets/fuzz_printf.rs b/fuzz/fuzz_targets/fuzz_printf.rs index 72fac540b17..e8d74e2bedd 100644 --- a/fuzz/fuzz_targets/fuzz_printf.rs +++ b/fuzz/fuzz_targets/fuzz_printf.rs @@ -8,8 +8,9 @@ use libfuzzer_sys::fuzz_target; use uu_printf::uumain; -use rand::seq::SliceRandom; use rand::Rng; +use rand::seq::IndexedRandom; +use std::env; use std::ffi::OsString; mod fuzz_common; @@ -43,34 +44,34 @@ fn generate_escape_sequence(rng: &mut impl Rng) -> String { } fn generate_printf() -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let format_specifiers = ["%s", "%d", "%f", "%x", "%o", "%c", "%b", "%q"]; let mut printf_str = String::new(); // Add a 20% chance of generating an invalid format specifier - if rng.gen_bool(0.2) { + if rng.random_bool(0.2) { printf_str.push_str("%z"); // Invalid format specifier } else { let specifier = *format_specifiers.choose(&mut rng).unwrap(); printf_str.push_str(specifier); // Add a 20% chance of introducing complex format strings - if rng.gen_bool(0.2) { - printf_str.push_str(&format!(" %{}", rng.gen_range(1..=1000))); + if rng.random_bool(0.2) { + printf_str.push_str(&format!(" %{}", rng.random_range(1..=1000))); } else { // Add a random string or number after the specifier if specifier == "%s" { printf_str.push_str(&format!( " {}", - generate_random_string(rng.gen_range(1..=10)) + generate_random_string(rng.random_range(1..=10)) )); } else { - printf_str.push_str(&format!(" {}", rng.gen_range(1..=1000))); + printf_str.push_str(&format!(" {}", rng.random_range(1..=1000))); } } } // Add a 10% chance of including an escape sequence - if rng.gen_bool(0.1) { + if rng.random_bool(0.1) { printf_str.push_str(&generate_escape_sequence(&mut rng)); } printf_str @@ -82,6 +83,10 @@ fuzz_target!(|_data: &[u8]| { args.extend(printf_input.split_whitespace().map(OsString::from)); let rust_result = generate_and_run_uumain(&args, uumain, None); + // TODO remove once uutils printf supports localization + unsafe { + env::set_var("LC_ALL", "C"); + } let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { Ok(result) => result, Err(error_result) => { diff --git a/fuzz/fuzz_targets/fuzz_seq.rs b/fuzz/fuzz_targets/fuzz_seq.rs index 7bb4f8af956..d36f0720a65 100644 --- a/fuzz/fuzz_targets/fuzz_seq.rs +++ b/fuzz/fuzz_targets/fuzz_seq.rs @@ -19,23 +19,23 @@ use crate::fuzz_common::{ static CMD_PATH: &str = "seq"; fn generate_seq() -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); // Generate 1 to 3 numbers for seq arguments - let arg_count = rng.gen_range(1..=3); + let arg_count = rng.random_range(1..=3); let mut args = Vec::new(); for _ in 0..arg_count { - if rng.gen_ratio(1, 100) { + if rng.random_ratio(1, 100) { // 1% chance to add a random string - args.push(generate_random_string(rng.gen_range(1..=10))); + args.push(generate_random_string(rng.random_range(1..=10))); } else { // 99% chance to add a numeric value - match rng.gen_range(0..=3) { - 0 => args.push(rng.gen_range(-10000..=10000).to_string()), // Large or small integers - 1 => args.push(rng.gen_range(-100.0..100.0).to_string()), // Floating-point numbers - 2 => args.push(rng.gen_range(-100..0).to_string()), // Negative integers - _ => args.push(rng.gen_range(1..=100).to_string()), // Regular integers + match rng.random_range(0..=3) { + 0 => args.push(rng.random_range(-10000..=10000).to_string()), // Large or small integers + 1 => args.push(rng.random_range(-100.0..100.0).to_string()), // Floating-point numbers + 2 => args.push(rng.random_range(-100..0).to_string()), // Negative integers + _ => args.push(rng.random_range(1..=100).to_string()), // Regular integers } } } diff --git a/fuzz/fuzz_targets/fuzz_seq_parse_number.rs b/fuzz/fuzz_targets/fuzz_seq_parse_number.rs new file mode 100644 index 00000000000..04da6d47f99 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_seq_parse_number.rs @@ -0,0 +1,15 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +#![no_main] + +use libfuzzer_sys::fuzz_target; +use std::str::FromStr; +use uu_seq::number::PreciseNumber; + +fuzz_target!(|data: &[u8]| { + if let Ok(s) = std::str::from_utf8(data) { + let _ = PreciseNumber::from_str(s); + } +}); diff --git a/fuzz/fuzz_targets/fuzz_sort.rs b/fuzz/fuzz_targets/fuzz_sort.rs index 3520bbaefed..e94938c3903 100644 --- a/fuzz/fuzz_targets/fuzz_sort.rs +++ b/fuzz/fuzz_targets/fuzz_sort.rs @@ -20,18 +20,18 @@ use crate::fuzz_common::{ static CMD_PATH: &str = "sort"; fn generate_sort_args() -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); - let arg_count = rng.gen_range(1..=5); + let arg_count = rng.random_range(1..=5); let mut args = Vec::new(); for _ in 0..arg_count { - match rng.gen_range(0..=4) { + match rng.random_range(0..=4) { 0 => args.push(String::from("-r")), // Reverse the result of comparisons 1 => args.push(String::from("-n")), // Compare according to string numerical value 2 => args.push(String::from("-f")), // Fold lower case to upper case characters - 3 => args.push(generate_random_string(rng.gen_range(1..=10))), // Random string (to simulate file names) - _ => args.push(String::from("-k") + &rng.gen_range(1..=5).to_string()), // Sort via a specified field + 3 => args.push(generate_random_string(rng.random_range(1..=10))), // Random string (to simulate file names) + _ => args.push(String::from("-k") + &rng.random_range(1..=5).to_string()), // Sort via a specified field } } @@ -39,11 +39,11 @@ fn generate_sort_args() -> String { } fn generate_random_lines(count: usize) -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let mut lines = Vec::new(); for _ in 0..count { - lines.push(generate_random_string(rng.gen_range(1..=20))); + lines.push(generate_random_string(rng.random_range(1..=20))); } lines.join("\n") @@ -60,7 +60,9 @@ fuzz_target!(|_data: &[u8]| { let rust_result = generate_and_run_uumain(&args, uumain, Some(&input_lines)); // TODO remove once uutils sort supports localization - env::set_var("LC_COLLATE", "C"); + unsafe { + env::set_var("LC_ALL", "C"); + } let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, Some(&input_lines)) { Ok(result) => result, Err(error_result) => { diff --git a/fuzz/fuzz_targets/fuzz_split.rs b/fuzz/fuzz_targets/fuzz_split.rs index 876c8dd21d4..9a925b222ad 100644 --- a/fuzz/fuzz_targets/fuzz_split.rs +++ b/fuzz/fuzz_targets/fuzz_split.rs @@ -13,18 +13,18 @@ use std::ffi::OsString; mod fuzz_common; use crate::fuzz_common::{ - compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, CommandResult, + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, }; static CMD_PATH: &str = "split"; fn generate_split_args() -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let mut args = Vec::new(); - match rng.gen_range(0..=9) { + match rng.random_range(0..=9) { 0 => { args.push(String::from("-a")); // Suffix length - args.push(rng.gen_range(1..=8).to_string()); + args.push(rng.random_range(1..=8).to_string()); } 1 => { args.push(String::from("--additional-suffix")); @@ -32,17 +32,17 @@ fn generate_split_args() -> String { } 2 => { args.push(String::from("-b")); // Bytes per output file - args.push(rng.gen_range(1..=1024).to_string() + "K"); + args.push(rng.random_range(1..=1024).to_string() + "K"); } 3 => { args.push(String::from("-C")); // Line bytes - args.push(rng.gen_range(1..=1024).to_string()); + args.push(rng.random_range(1..=1024).to_string()); } 4 => args.push(String::from("-d")), // Use numeric suffixes 5 => args.push(String::from("-x")), // Use hex suffixes 6 => { args.push(String::from("-l")); // Number of lines per output file - args.push(rng.gen_range(1..=1000).to_string()); + args.push(rng.random_range(1..=1000).to_string()); } 7 => { args.push(String::from("--filter")); @@ -61,11 +61,11 @@ fn generate_split_args() -> String { // Function to generate a random string of lines fn generate_random_lines(count: usize) -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let mut lines = Vec::new(); for _ in 0..count { - lines.push(generate_random_string(rng.gen_range(1..=20))); + lines.push(generate_random_string(rng.random_range(1..=20))); } lines.join("\n") diff --git a/fuzz/fuzz_targets/fuzz_test.rs b/fuzz/fuzz_targets/fuzz_test.rs index bed7ca77088..39926b26f76 100644 --- a/fuzz/fuzz_targets/fuzz_test.rs +++ b/fuzz/fuzz_targets/fuzz_test.rs @@ -8,8 +8,8 @@ use libfuzzer_sys::fuzz_target; use uu_test::uumain; -use rand::seq::SliceRandom; use rand::Rng; +use rand::prelude::IndexedRandom; use std::ffi::OsString; mod fuzz_common; @@ -18,6 +18,7 @@ use crate::fuzz_common::{ compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, }; +#[allow(clippy::upper_case_acronyms)] #[derive(PartialEq, Debug, Clone)] enum ArgType { STRING, @@ -38,7 +39,7 @@ struct TestArg { } fn generate_random_path(rng: &mut dyn rand::RngCore) -> &'static str { - match rng.gen_range(0..=3) { + match rng.random_range(0..=3) { 0 => "/dev/null", 1 => "/dev/random", 2 => "/tmp", @@ -64,6 +65,14 @@ fn generate_test_args() -> Vec { arg: "!=".to_string(), arg_type: ArgType::STRINGSTRING, }, + TestArg { + arg: ">".to_string(), + arg_type: ArgType::STRINGSTRING, + }, + TestArg { + arg: "<".to_string(), + arg_type: ArgType::STRINGSTRING, + }, TestArg { arg: "-eq".to_string(), arg_type: ArgType::INTEGERINTEGER, @@ -112,15 +121,15 @@ fn generate_test_args() -> Vec { } fn generate_test_arg() -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let test_args = generate_test_args(); let mut arg = String::new(); - let choice = rng.gen_range(0..=5); + let choice = rng.random_range(0..=5); match choice { 0 => { - arg.push_str(&rng.gen_range(-100..=100).to_string()); + arg.push_str(&rng.random_range(-100..=100).to_string()); } 1..=3 => { let test_arg = test_args @@ -129,32 +138,32 @@ fn generate_test_arg() -> String { if test_arg.arg_type == ArgType::INTEGER { arg.push_str(&format!( "{} {} {}", - &rng.gen_range(-100..=100).to_string(), + rng.random_range(-100..=100).to_string(), test_arg.arg, - &rng.gen_range(-100..=100).to_string() + rng.random_range(-100..=100).to_string() )); } else if test_arg.arg_type == ArgType::STRINGSTRING { - let random_str = generate_random_string(rng.gen_range(1..=10)); - let random_str2 = generate_random_string(rng.gen_range(1..=10)); + let random_str = generate_random_string(rng.random_range(1..=10)); + let random_str2 = generate_random_string(rng.random_range(1..=10)); arg.push_str(&format!( - "{} {} {}", - &random_str, test_arg.arg, &random_str2 + "{random_str} {} {random_str2}", + test_arg.arg, )); } else if test_arg.arg_type == ArgType::STRING { - let random_str = generate_random_string(rng.gen_range(1..=10)); - arg.push_str(&format!("{} {}", test_arg.arg, &random_str)); + let random_str = generate_random_string(rng.random_range(1..=10)); + arg.push_str(&format!("{} {random_str}", test_arg.arg)); } else if test_arg.arg_type == ArgType::FILEFILE { let path = generate_random_path(&mut rng); let path2 = generate_random_path(&mut rng); - arg.push_str(&format!("{} {} {}", path, test_arg.arg, path2)); + arg.push_str(&format!("{path} {} {path2}", test_arg.arg)); } else if test_arg.arg_type == ArgType::FILE { let path = generate_random_path(&mut rng); - arg.push_str(&format!("{} {}", test_arg.arg, path)); + arg.push_str(&format!("{} {path}", test_arg.arg)); } } 4 => { - let random_str = generate_random_string(rng.gen_range(1..=10)); + let random_str = generate_random_string(rng.random_range(1..=10)); arg.push_str(&random_str); } _ => { @@ -167,7 +176,7 @@ fn generate_test_arg() -> String { .collect(); if let Some(test_arg) = file_test_args.choose(&mut rng) { - arg.push_str(&format!("{}{}", test_arg.arg, path)); + arg.push_str(&format!("{}{path}", test_arg.arg)); } } } @@ -176,8 +185,8 @@ fn generate_test_arg() -> String { } fuzz_target!(|_data: &[u8]| { - let mut rng = rand::thread_rng(); - let max_args = rng.gen_range(1..=6); + let mut rng = rand::rng(); + let max_args = rng.random_range(1..=6); let mut args = vec![OsString::from("test")]; for _ in 0..max_args { diff --git a/fuzz/fuzz_targets/fuzz_tr.rs b/fuzz/fuzz_targets/fuzz_tr.rs new file mode 100644 index 00000000000..d260e378088 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_tr.rs @@ -0,0 +1,73 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +#![no_main] +use libfuzzer_sys::fuzz_target; +use std::ffi::OsString; +use uu_tr::uumain; + +use rand::Rng; + +mod fuzz_common; +use crate::fuzz_common::{ + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, +}; +static CMD_PATH: &str = "tr"; + +fn generate_tr_args() -> Vec { + let mut rng = rand::rng(); + let mut args = Vec::new(); + + // Translate, squeeze, and/or delete characters + let opts = ["-c", "-d", "-s", "-t"]; + for opt in &opts { + if rng.random_bool(0.25) { + args.push(opt.to_string()); + } + } + + // Generating STRING1 and optionally STRING2 + let string1 = generate_random_string(rng.random_range(1..=20)); + args.push(string1); + if rng.random_bool(0.7) { + // Higher chance to add STRING2 for translation + let string2 = generate_random_string(rng.random_range(1..=20)); + args.push(string2); + } + + args +} + +fuzz_target!(|_data: &[u8]| { + let tr_args = generate_tr_args(); + let mut args = vec![OsString::from("tr")]; + args.extend(tr_args.iter().map(OsString::from)); + + let input_chars = generate_random_string(100); + + let rust_result = generate_and_run_uumain(&args, uumain, Some(&input_chars)); + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, Some(&input_chars)) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "tr", + &format!("{:?}", &args[1..]), + Some(&input_chars), + &rust_result, + &gnu_result, + false, + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_wc.rs b/fuzz/fuzz_targets/fuzz_wc.rs index dc85bbc3541..39dfb1ee862 100644 --- a/fuzz/fuzz_targets/fuzz_wc.rs +++ b/fuzz/fuzz_targets/fuzz_wc.rs @@ -13,21 +13,21 @@ use std::ffi::OsString; mod fuzz_common; use crate::fuzz_common::{ - compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, CommandResult, + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, }; static CMD_PATH: &str = "wc"; fn generate_wc_args() -> String { - let mut rng = rand::thread_rng(); - let arg_count = rng.gen_range(1..=6); + let mut rng = rand::rng(); + let arg_count = rng.random_range(1..=6); let mut args = Vec::new(); for _ in 0..arg_count { // Introduce a chance to add invalid arguments - if rng.gen_bool(0.1) { - args.push(generate_random_string(rng.gen_range(1..=20))); + if rng.random_bool(0.1) { + args.push(generate_random_string(rng.random_range(1..=20))); } else { - match rng.gen_range(0..=5) { + match rng.random_range(0..=5) { 0 => args.push(String::from("-c")), 1 => args.push(String::from("-m")), 2 => args.push(String::from("-l")), @@ -36,7 +36,7 @@ fn generate_wc_args() -> String { // TODO 5 => { args.push(String::from("--files0-from")); - if rng.gen_bool(0.5) { + if rng.random_bool(0.5) { args.push(generate_random_string(50)); // Longer invalid file name } else { args.push(generate_random_string(5)); @@ -52,14 +52,14 @@ fn generate_wc_args() -> String { // Function to generate a random string of lines, including invalid ones fn generate_random_lines(count: usize) -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let mut lines = Vec::new(); for _ in 0..count { - if rng.gen_bool(0.1) { - lines.push(generate_random_string(rng.gen_range(1000..=5000))); // Very long invalid line + if rng.random_bool(0.1) { + lines.push(generate_random_string(rng.random_range(1000..=5000))); // Very long invalid line } else { - lines.push(generate_random_string(rng.gen_range(1..=20))); + lines.push(generate_random_string(rng.random_range(1..=20))); } } diff --git a/renovate.json b/renovate.json index 9dc8c691977..8f60343a8ef 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,5 @@ { "extends": [ - "config:base" + "config:recommended" ] } diff --git a/src/bin/coreutils.rs b/src/bin/coreutils.rs index fc2cd16add2..b29e7ea2337 100644 --- a/src/bin/coreutils.rs +++ b/src/bin/coreutils.rs @@ -21,7 +21,10 @@ include!(concat!(env!("OUT_DIR"), "/uutils_map.rs")); fn usage(utils: &UtilityMap, name: &str) { println!("{name} {VERSION} (multi-call binary)\n"); - println!("Usage: {name} [function [arguments...]]\n"); + println!("Usage: {name} [function [arguments...]]"); + println!(" {name} --list\n"); + println!("Options:"); + println!(" --list lists all defined functions, one per row\n"); println!("Currently defined functions:\n"); #[allow(clippy::map_clone)] let mut utils: Vec<&str> = utils.keys().map(|&s| s).collect(); @@ -34,6 +37,8 @@ fn usage(utils: &UtilityMap, name: &str) { ); } +/// # Panics +/// Panics if the binary path cannot be determined fn binary_path(args: &mut impl Iterator) -> PathBuf { match args.next() { Some(ref s) if !s.is_empty() => PathBuf::from(s), @@ -60,7 +65,7 @@ fn main() { // binary name equals util name? if let Some(&(uumain, _)) = utils.get(binary_as_util) { - process::exit(uumain((vec![binary.into()].into_iter()).chain(args))); + process::exit(uumain(vec![binary.into()].into_iter().chain(args))); } // binary name equals prefixed util name? @@ -85,36 +90,42 @@ fn main() { process::exit(1); } - let util = match util_os.to_str() { - Some(util) => util, - None => not_found(&util_os), + let Some(util) = util_os.to_str() else { + not_found(&util_os) }; - if util == "completion" { - gen_completions(args, &utils); - } - - if util == "manpage" { - gen_manpage(args, &utils); + match util { + "completion" => gen_completions(args, &utils), + "manpage" => gen_manpage(args, &utils), + "--list" => { + let mut utils: Vec<_> = utils.keys().collect(); + utils.sort(); + for util in utils { + println!("{util}"); + } + process::exit(0); + } + // Not a special command: fallthrough to calling a util + _ => {} } match utils.get(util) { Some(&(uumain, _)) => { - process::exit(uumain((vec![util_os].into_iter()).chain(args))); + process::exit(uumain(vec![util_os].into_iter().chain(args))); } None => { if util == "--help" || util == "-h" { // see if they want help on a specific util if let Some(util_os) = args.next() { - let util = match util_os.to_str() { - Some(util) => util, - None => not_found(&util_os), + let Some(util) = util_os.to_str() else { + not_found(&util_os) }; match utils.get(util) { Some(&(uumain, _)) => { let code = uumain( - (vec![util_os, OsString::from("--help")].into_iter()) + vec![util_os, OsString::from("--help")] + .into_iter() .chain(args), ); io::stdout().flush().expect("could not flush stdout"); @@ -138,6 +149,8 @@ fn main() { } /// Prints completions for the utility in the first parameter for the shell in the second parameter to stdout +/// # Panics +/// Panics if the utility map is empty fn gen_completions( args: impl Iterator, util_map: &UtilityMap, @@ -176,6 +189,8 @@ fn gen_completions( } /// Generate the manpage for the utility in the first parameter +/// # Panics +/// Panics if the utility map is empty fn gen_manpage( args: impl Iterator, util_map: &UtilityMap, @@ -208,6 +223,8 @@ fn gen_manpage( process::exit(0); } +/// # Panics +/// Panics if the utility map is empty fn gen_coreutils_app(util_map: &UtilityMap) -> Command { let mut command = Command::new("coreutils"); for (name, (_, sub_app)) in util_map { diff --git a/src/bin/uudoc.rs b/src/bin/uudoc.rs index 77c7a2fcfdd..6a215a4ada4 100644 --- a/src/bin/uudoc.rs +++ b/src/bin/uudoc.rs @@ -13,6 +13,9 @@ use zip::ZipArchive; include!(concat!(env!("OUT_DIR"), "/uutils_map.rs")); +/// # Errors +/// Returns an error if the writer fails. +#[allow(clippy::too_many_lines)] fn main() -> io::Result<()> { let mut tldr_zip = File::open("docs/tldr.zip") .ok() @@ -20,7 +23,9 @@ fn main() -> io::Result<()> { if tldr_zip.is_none() { println!("Warning: No tldr archive found, so the documentation will not include examples."); - println!("To include examples in the documentation, download the tldr archive and put it in the docs/ folder."); + println!( + "To include examples in the documentation, download the tldr archive and put it in the docs/ folder." + ); println!(); println!(" curl https://tldr.sh/assets/tldr.zip -o docs/tldr.zip"); println!(); @@ -43,7 +48,9 @@ fn main() -> io::Result<()> { * [Installation](installation.md)\n\ * [Build from source](build.md)\n\ * [Platform support](platforms.md)\n\ - * [Contributing](contributing.md)\n\ + * [Contributing](CONTRIBUTING.md)\n\ + \t* [Development](DEVELOPMENT.md)\n\ + \t* [Code of Conduct](CODE_OF_CONDUCT.md)\n\ * [GNU test coverage](test_coverage.md)\n\ * [Extensions](extensions.md)\n\ \n\ @@ -57,7 +64,7 @@ fn main() -> io::Result<()> { for platform in ["unix", "macos", "windows", "unix_android"] { let platform_utils: Vec = String::from_utf8( std::process::Command::new("./util/show-utils.sh") - .arg(format!("--features=feat_os_{}", platform)) + .arg(format!("--features=feat_os_{platform}")) .output()? .stdout, ) @@ -107,7 +114,7 @@ fn main() -> io::Result<()> { "| util | Linux | macOS | Windows | FreeBSD | Android |\n\ | ---------------- | ----- | ----- | ------- | ------- | ------- |" )?; - for (&name, _) in &utils { + for &(&name, _) in &utils { if name == "[" { continue; } @@ -131,7 +138,7 @@ fn main() -> io::Result<()> { if name == "[" { continue; } - let p = format!("docs/src/utils/{}.md", name); + let p = format!("docs/src/utils/{name}.md"); let markdown = File::open(format!("src/uu/{name}/{name}.md")) .and_then(|mut f: File| { @@ -151,11 +158,11 @@ fn main() -> io::Result<()> { markdown, } .markdown()?; - println!("Wrote to '{}'", p); + println!("Wrote to '{p}'"); } else { - println!("Error writing to {}", p); + println!("Error writing to {p}"); } - writeln!(summary, "* [{0}](utils/{0}.md)", name)?; + writeln!(summary, "* [{name}](utils/{name}.md)")?; } Ok(()) } @@ -169,7 +176,9 @@ struct MDWriter<'a, 'b> { markdown: Option, } -impl<'a, 'b> MDWriter<'a, 'b> { +impl MDWriter<'_, '_> { + /// # Errors + /// Returns an error if the writer fails. fn markdown(&mut self) -> io::Result<()> { write!(self.w, "# {}\n\n", self.name)?; self.additional()?; @@ -180,6 +189,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { self.examples() } + /// # Errors + /// Returns an error if the writer fails. fn additional(&mut self) -> io::Result<()> { writeln!(self.w, "
")?; self.platforms()?; @@ -187,6 +198,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { writeln!(self.w, "
") } + /// # Errors + /// Returns an error if the writer fails. fn platforms(&mut self) -> io::Result<()> { writeln!(self.w, "
")?; for (feature, icon) in [ @@ -201,7 +214,7 @@ impl<'a, 'b> MDWriter<'a, 'b> { .iter() .any(|u| u == self.name) { - writeln!(self.w, "", icon)?; + writeln!(self.w, "")?; } } writeln!(self.w, "
")?; @@ -209,6 +222,10 @@ impl<'a, 'b> MDWriter<'a, 'b> { Ok(()) } + /// # Errors + /// Returns an error if the writer fails. + /// # Panics + /// Panics if the version is not found. fn version(&mut self) -> io::Result<()> { writeln!( self.w, @@ -217,19 +234,23 @@ impl<'a, 'b> MDWriter<'a, 'b> { ) } + /// # Errors + /// Returns an error if the writer fails. fn usage(&mut self) -> io::Result<()> { if let Some(markdown) = &self.markdown { let usage = uuhelp_parser::parse_usage(markdown); let usage = usage.replace("{}", self.name); writeln!(self.w, "\n```")?; - writeln!(self.w, "{}", usage)?; + writeln!(self.w, "{usage}")?; writeln!(self.w, "```") } else { Ok(()) } } + /// # Errors + /// Returns an error if the writer fails. fn about(&mut self) -> io::Result<()> { if let Some(markdown) = &self.markdown { writeln!(self.w, "{}", uuhelp_parser::parse_about(markdown)) @@ -238,6 +259,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { } } + /// # Errors + /// Returns an error if the writer fails. fn after_help(&mut self) -> io::Result<()> { if let Some(markdown) = &self.markdown { if let Some(after_help) = uuhelp_parser::parse_section("after help", markdown) { @@ -248,6 +271,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { Ok(()) } + /// # Errors + /// Returns an error if the writer fails. fn examples(&mut self) -> io::Result<()> { if let Some(zip) = self.tldr_zip { let content = if let Some(f) = @@ -268,14 +293,14 @@ impl<'a, 'b> MDWriter<'a, 'b> { writeln!(self.w)?; for line in content.lines().skip_while(|l| !l.starts_with('-')) { if let Some(l) = line.strip_prefix("- ") { - writeln!(self.w, "{}", l)?; + writeln!(self.w, "{l}")?; } else if line.starts_with('`') { writeln!(self.w, "```shell\n{}\n```", line.trim_matches('`'))?; } else if line.is_empty() { writeln!(self.w)?; } else { println!("Not sure what to do with this line:"); - println!("{}", line); + println!("{line}"); } } writeln!(self.w)?; @@ -292,6 +317,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { Ok(()) } + /// # Errors + /// Returns an error if the writer fails. fn options(&mut self) -> io::Result<()> { writeln!(self.w, "

Options

")?; write!(self.w, "
")?; @@ -305,14 +332,14 @@ impl<'a, 'b> MDWriter<'a, 'b> { write!(self.w, ", ")?; } write!(self.w, "")?; - write!(self.w, "--{}", l)?; + write!(self.w, "--{l}")?; if let Some(names) = arg.get_value_names() { write!( self.w, "={}", names .iter() - .map(|x| format!("<{}>", x)) + .map(|x| format!("<{x}>")) .collect::>() .join(" ") )?; @@ -326,14 +353,14 @@ impl<'a, 'b> MDWriter<'a, 'b> { write!(self.w, ", ")?; } write!(self.w, "")?; - write!(self.w, "-{}", s)?; + write!(self.w, "-{s}")?; if let Some(names) = arg.get_value_names() { write!( self.w, " {}", names .iter() - .map(|x| format!("<{}>", x)) + .map(|x| format!("<{x}>")) .collect::>() .join(" ") )?; @@ -354,6 +381,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { } } +/// # Panics +/// Panics if the archive is not ok fn get_zip_content(archive: &mut ZipArchive, name: &str) -> Option { let mut s = String::new(); archive.by_name(name).ok()?.read_to_string(&mut s).unwrap(); diff --git a/src/uu/arch/Cargo.toml b/src/uu/arch/Cargo.toml index b4d07b26c59..611aa6845cf 100644 --- a/src/uu/arch/Cargo.toml +++ b/src/uu/arch/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_arch" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "arch ~ (uutils) display machine architecture" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/arch" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/arch.rs" diff --git a/src/uu/arch/src/arch.rs b/src/uu/arch/src/arch.rs index 0d71a818379..590def48fb9 100644 --- a/src/uu/arch/src/arch.rs +++ b/src/uu/arch/src/arch.rs @@ -5,7 +5,7 @@ use platform_info::*; -use clap::{crate_version, Command}; +use clap::Command; use uucore::error::{UResult, USimpleError}; use uucore::{help_about, help_section}; @@ -24,7 +24,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .after_help(SUMMARY) .infer_long_args(true) diff --git a/src/uu/base32/Cargo.toml b/src/uu/base32/Cargo.toml index 1c27e14cf3f..42421311c0e 100644 --- a/src/uu/base32/Cargo.toml +++ b/src/uu/base32/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_base32" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "base32 ~ (uutils) decode/encode input (base32-encoding)" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/base32" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/base32.rs" diff --git a/src/uu/base32/src/base32.rs b/src/uu/base32/src/base32.rs index 09250421c25..e14e83921e2 100644 --- a/src/uu/base32/src/base32.rs +++ b/src/uu/base32/src/base32.rs @@ -3,13 +3,12 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -use std::io::{stdin, Read}; +pub mod base_common; +use base_common::ReadSeek; use clap::Command; use uucore::{encoding::Format, error::UResult, help_about, help_usage}; -pub mod base_common; - const ABOUT: &str = help_about!("base32.md"); const USAGE: &str = help_usage!("base32.md"); @@ -17,20 +16,11 @@ const USAGE: &str = help_usage!("base32.md"); pub fn uumain(args: impl uucore::Args) -> UResult<()> { let format = Format::Base32; - let config: base_common::Config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; + let config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; - // Create a reference to stdin so we can return a locked stdin from - // parse_base_cmd_args - let stdin_raw = stdin(); - let mut input: Box = base_common::get_input(&config, &stdin_raw)?; + let mut input: Box = base_common::get_input(&config)?; - base_common::handle_input( - &mut input, - format, - config.wrap_cols, - config.ignore_garbage, - config.decode, - ) + base_common::handle_input(&mut input, format, config) } pub fn uu_app() -> Command { diff --git a/src/uu/base32/src/base_common.rs b/src/uu/base32/src/base_common.rs index 68c40287db7..05bfc89b28f 100644 --- a/src/uu/base32/src/base_common.rs +++ b/src/uu/base32/src/base_common.rs @@ -3,27 +3,35 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -use std::io::{stdout, Read, Write}; +// spell-checker:ignore hexupper lsbf msbf unpadded nopad aGVsbG8sIHdvcmxkIQ +use clap::{Arg, ArgAction, Command}; +use std::fs::File; +use std::io::{self, ErrorKind, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; use uucore::display::Quotable; -use uucore::encoding::{wrap_print, Data, EncodeError, Format}; +use uucore::encoding::{ + BASE2LSBF, BASE2MSBF, Format, Z85Wrapper, + for_base_common::{BASE32, BASE32HEX, BASE64, BASE64_NOPAD, BASE64URL, HEXUPPER_PERMISSIVE}, +}; +use uucore::encoding::{EncodingWrapper, SupportsFastDecodeAndEncode}; use uucore::error::{FromIo, UResult, USimpleError, UUsageError}; use uucore::format_usage; -use std::fs::File; -use std::io::{BufReader, Stdin}; -use std::path::Path; - -use clap::{crate_version, Arg, ArgAction, Command}; +pub const BASE_CMD_PARSE_ERROR: i32 = 1; -pub static BASE_CMD_PARSE_ERROR: i32 = 1; +/// Encoded output will be formatted in lines of this length (the last line can be shorter) +/// +/// Other implementations default to 76 +/// +/// This default is only used if no "-w"/"--wrap" argument is passed +pub const WRAP_DEFAULT: usize = 76; -// Config. pub struct Config { pub decode: bool, pub ignore_garbage: bool, pub wrap_cols: Option, - pub to_read: Option, + pub to_read: Option, } pub mod options { @@ -35,32 +43,36 @@ pub mod options { impl Config { pub fn from(options: &clap::ArgMatches) -> UResult { - let file: Option = match options.get_many::(options::FILE) { + let to_read = match options.get_many::(options::FILE) { Some(mut values) => { let name = values.next().unwrap(); + if let Some(extra_op) = values.next() { return Err(UUsageError::new( BASE_CMD_PARSE_ERROR, - format!("extra operand {}", extra_op.quote(),), + format!("extra operand {}", extra_op.quote()), )); } if name == "-" { None } else { - if !Path::exists(Path::new(name)) { + let path = Path::new(name); + + if !path.exists() { return Err(USimpleError::new( BASE_CMD_PARSE_ERROR, - format!("{}: No such file or directory", name.maybe_quote()), + format!("{}: No such file or directory", path.maybe_quote()), )); } - Some(name.clone()) + + Some(path.to_owned()) } } None => None, }; - let cols = options + let wrap_cols = options .get_one::(options::WRAP) .map(|num| { num.parse::().map_err(|_| { @@ -75,8 +87,8 @@ impl Config { Ok(Self { decode: options.get_flag(options::DECODE), ignore_garbage: options.get_flag(options::IGNORE_GARBAGE), - wrap_cols: cols, - to_read: file, + wrap_cols, + to_read, }) } } @@ -92,7 +104,7 @@ pub fn parse_base_cmd_args( pub fn base_app(about: &'static str, usage: &str) -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(about) .override_usage(format_usage(usage)) .infer_long_args(true) @@ -100,85 +112,735 @@ pub fn base_app(about: &'static str, usage: &str) -> Command { .arg( Arg::new(options::DECODE) .short('d') + .visible_short_alias('D') .long(options::DECODE) .help("decode data") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with(options::DECODE), ) .arg( Arg::new(options::IGNORE_GARBAGE) .short('i') .long(options::IGNORE_GARBAGE) .help("when decoding, ignore non-alphabetic characters") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with(options::IGNORE_GARBAGE), ) .arg( Arg::new(options::WRAP) .short('w') .long(options::WRAP) .value_name("COLS") - .help( - "wrap encoded lines after COLS character (default 76, 0 to disable wrapping)", - ), + .help(format!("wrap encoded lines after COLS character (default {WRAP_DEFAULT}, 0 to disable wrapping)")) + .overrides_with(options::WRAP), ) // "multiple" arguments are used to check whether there is more than one // file passed in. .arg( Arg::new(options::FILE) .index(1) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) .value_hint(clap::ValueHint::FilePath), ) } -pub fn get_input<'a>(config: &Config, stdin_ref: &'a Stdin) -> UResult> { +/// A trait alias for types that implement both `Read` and `Seek`. +pub trait ReadSeek: Read + Seek {} + +/// Automatically implement the `ReadSeek` trait for any type that implements both `Read` and `Seek`. +impl ReadSeek for T {} + +pub fn get_input(config: &Config) -> UResult> { match &config.to_read { - Some(name) => { - let file_buf = - File::open(Path::new(name)).map_err_context(|| name.maybe_quote().to_string())?; - Ok(Box::new(BufReader::new(file_buf))) // as Box + Some(path_buf) => { + // Do not buffer input, because buffering is handled by `fast_decode` and `fast_encode` + let file = + File::open(path_buf).map_err_context(|| path_buf.maybe_quote().to_string())?; + Ok(Box::new(file)) } None => { - Ok(Box::new(stdin_ref.lock())) // as Box + let mut buffer = Vec::new(); + io::stdin().read_to_end(&mut buffer)?; + Ok(Box::new(io::Cursor::new(buffer))) } } } -pub fn handle_input( - input: &mut R, +/// Determines if the input buffer ends with padding ('=') after trimming trailing whitespace. +fn has_padding(input: &mut R) -> UResult { + let mut buf = Vec::new(); + input + .read_to_end(&mut buf) + .map_err(|err| USimpleError::new(1, format_read_error(err.kind())))?; + + // Reverse iterator and skip trailing whitespace without extra collections + let has_padding = buf + .iter() + .rfind(|&&byte| !byte.is_ascii_whitespace()) + .is_some_and(|&byte| byte == b'='); + + input.seek(SeekFrom::Start(0))?; + Ok(has_padding) +} + +pub fn handle_input(input: &mut R, format: Format, config: Config) -> UResult<()> { + let has_padding = has_padding(input)?; + + let supports_fast_decode_and_encode = + get_supports_fast_decode_and_encode(format, config.decode, has_padding); + + let supports_fast_decode_and_encode_ref = supports_fast_decode_and_encode.as_ref(); + + let mut stdout_lock = io::stdout().lock(); + + if config.decode { + fast_decode::fast_decode( + input, + &mut stdout_lock, + supports_fast_decode_and_encode_ref, + config.ignore_garbage, + ) + } else { + fast_encode::fast_encode( + input, + &mut stdout_lock, + supports_fast_decode_and_encode_ref, + config.wrap_cols, + ) + } +} + +pub fn get_supports_fast_decode_and_encode( format: Format, - line_wrap: Option, - ignore_garbage: bool, decode: bool, -) -> UResult<()> { - let mut data = Data::new(input, format).ignore_garbage(ignore_garbage); - if let Some(wrap) = line_wrap { - data = data.line_wrap(wrap); - } - - if decode { - match data.decode() { - Ok(s) => { - // Silent the warning as we want to the error message - #[allow(clippy::question_mark)] - if stdout().write_all(&s).is_err() { - // on windows console, writing invalid utf8 returns an error - return Err(USimpleError::new(1, "error: cannot write non-utf8 data")); + has_padding: bool, +) -> Box { + const BASE16_VALID_DECODING_MULTIPLE: usize = 2; + const BASE2_VALID_DECODING_MULTIPLE: usize = 8; + const BASE32_VALID_DECODING_MULTIPLE: usize = 8; + const BASE64_VALID_DECODING_MULTIPLE: usize = 4; + + const BASE16_UNPADDED_MULTIPLE: usize = 1; + const BASE2_UNPADDED_MULTIPLE: usize = 1; + const BASE32_UNPADDED_MULTIPLE: usize = 5; + const BASE64_UNPADDED_MULTIPLE: usize = 3; + + match format { + Format::Base16 => Box::from(EncodingWrapper::new( + HEXUPPER_PERMISSIVE, + BASE16_VALID_DECODING_MULTIPLE, + BASE16_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"0123456789ABCDEFabcdef", + )), + Format::Base2Lsbf => Box::from(EncodingWrapper::new( + BASE2LSBF, + BASE2_VALID_DECODING_MULTIPLE, + BASE2_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"01", + )), + Format::Base2Msbf => Box::from(EncodingWrapper::new( + BASE2MSBF, + BASE2_VALID_DECODING_MULTIPLE, + BASE2_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"01", + )), + Format::Base32 => Box::from(EncodingWrapper::new( + BASE32, + BASE32_VALID_DECODING_MULTIPLE, + BASE32_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567=", + )), + Format::Base32Hex => Box::from(EncodingWrapper::new( + BASE32HEX, + BASE32_VALID_DECODING_MULTIPLE, + BASE32_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"0123456789ABCDEFGHIJKLMNOPQRSTUV=", + )), + Format::Base64 => { + let alphabet: &[u8] = if has_padding { + &b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/="[..] + } else { + &b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/"[..] + }; + let wrapper = if decode && !has_padding { + BASE64_NOPAD + } else { + BASE64 + }; + Box::from(EncodingWrapper::new( + wrapper, + BASE64_VALID_DECODING_MULTIPLE, + BASE64_UNPADDED_MULTIPLE, + alphabet, + )) + } + Format::Base64Url => Box::from(EncodingWrapper::new( + BASE64URL, + BASE64_VALID_DECODING_MULTIPLE, + BASE64_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789=_-", + )), + Format::Z85 => Box::from(Z85Wrapper {}), + } +} + +pub mod fast_encode { + use crate::base_common::{WRAP_DEFAULT, format_read_error}; + use std::{ + collections::VecDeque, + io::{self, ErrorKind, Read, Write}, + num::NonZeroUsize, + }; + use uucore::{ + encoding::SupportsFastDecodeAndEncode, + error::{UResult, USimpleError}, + }; + + struct LineWrapping { + line_length: NonZeroUsize, + print_buffer: Vec, + } + + // Start of helper functions + fn encode_in_chunks_to_buffer( + supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode, + encode_in_chunks_of_size: usize, + bytes_to_steal: usize, + read_buffer: &[u8], + encoded_buffer: &mut VecDeque, + leftover_buffer: &mut VecDeque, + ) -> UResult<()> { + let bytes_to_chunk = if bytes_to_steal > 0 { + let (stolen_bytes, rest_of_read_buffer) = read_buffer.split_at(bytes_to_steal); + + leftover_buffer.extend(stolen_bytes); + + // After appending the stolen bytes to `leftover_buffer`, it should be the right size + assert_eq!(leftover_buffer.len(), encode_in_chunks_of_size); + + // Encode the old unencoded data and the stolen bytes, and add the result to + // `encoded_buffer` + supports_fast_decode_and_encode + .encode_to_vec_deque(leftover_buffer.make_contiguous(), encoded_buffer)?; + + // Reset `leftover_buffer` + leftover_buffer.clear(); + + rest_of_read_buffer + } else { + // Do not need to steal bytes from `read_buffer` + read_buffer + }; + + let chunks_exact = bytes_to_chunk.chunks_exact(encode_in_chunks_of_size); + + let remainder = chunks_exact.remainder(); + + for sl in chunks_exact { + assert_eq!(sl.len(), encode_in_chunks_of_size); + + supports_fast_decode_and_encode.encode_to_vec_deque(sl, encoded_buffer)?; + } + + leftover_buffer.extend(remainder); + + Ok(()) + } + + fn write_without_line_breaks( + encoded_buffer: &mut VecDeque, + output: &mut dyn Write, + is_cleanup: bool, + empty_wrap: bool, + ) -> io::Result<()> { + // TODO + // `encoded_buffer` only has to be a VecDeque if line wrapping is enabled + // (`make_contiguous` should be a no-op here) + // Refactoring could avoid this call + output.write_all(encoded_buffer.make_contiguous())?; + + if is_cleanup { + if !empty_wrap { + output.write_all(b"\n")?; + } + } else { + encoded_buffer.clear(); + } + + Ok(()) + } + + fn write_with_line_breaks( + &mut LineWrapping { + ref line_length, + ref mut print_buffer, + }: &mut LineWrapping, + encoded_buffer: &mut VecDeque, + output: &mut dyn Write, + is_cleanup: bool, + ) -> io::Result<()> { + let line_length = line_length.get(); + + let make_contiguous_result = encoded_buffer.make_contiguous(); + + let chunks_exact = make_contiguous_result.chunks_exact(line_length); + + let mut bytes_added_to_print_buffer = 0; + + for sl in chunks_exact { + bytes_added_to_print_buffer += sl.len(); + + print_buffer.extend_from_slice(sl); + print_buffer.push(b'\n'); + } + + output.write_all(print_buffer)?; + + // Remove the bytes that were just printed from `encoded_buffer` + drop(encoded_buffer.drain(..bytes_added_to_print_buffer)); + + if is_cleanup { + if encoded_buffer.is_empty() { + // Do not write a newline in this case, because two trailing newlines should never be printed + } else { + // Print the partial line, since this is cleanup and no more data is coming + output.write_all(encoded_buffer.make_contiguous())?; + output.write_all(b"\n")?; + } + } else { + print_buffer.clear(); + } + + Ok(()) + } + + fn write_to_output( + line_wrapping: &mut Option, + encoded_buffer: &mut VecDeque, + output: &mut dyn Write, + is_cleanup: bool, + empty_wrap: bool, + ) -> io::Result<()> { + // Write all data in `encoded_buffer` to `output` + if let &mut Some(ref mut li) = line_wrapping { + write_with_line_breaks(li, encoded_buffer, output, is_cleanup)?; + } else { + write_without_line_breaks(encoded_buffer, output, is_cleanup, empty_wrap)?; + } + + Ok(()) + } + // End of helper functions + + pub fn fast_encode( + input: &mut dyn Read, + output: &mut dyn Write, + supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode, + wrap: Option, + ) -> UResult<()> { + // Based on performance testing + const INPUT_BUFFER_SIZE: usize = 32 * 1_024; + + const ENCODE_IN_CHUNKS_OF_SIZE_MULTIPLE: usize = 1_024; + + let encode_in_chunks_of_size = + supports_fast_decode_and_encode.unpadded_multiple() * ENCODE_IN_CHUNKS_OF_SIZE_MULTIPLE; + + assert!(encode_in_chunks_of_size > 0); + + // The "data-encoding" crate supports line wrapping, but not arbitrary line wrapping, only certain widths, so + // line wrapping must be handled here. + // https://github.com/ia0/data-encoding/blob/4f42ad7ef242f6d243e4de90cd1b46a57690d00e/lib/src/lib.rs#L1710 + let mut line_wrapping = match wrap { + // Line wrapping is disabled because "-w"/"--wrap" was passed with "0" + Some(0) => None, + // A custom line wrapping value was passed + Some(an) => Some(LineWrapping { + line_length: NonZeroUsize::new(an).unwrap(), + print_buffer: Vec::::new(), + }), + // Line wrapping was not set, so the default is used + None => Some(LineWrapping { + line_length: NonZeroUsize::new(WRAP_DEFAULT).unwrap(), + print_buffer: Vec::::new(), + }), + }; + + // Start of buffers + // Data that was read from `input` + let mut input_buffer = vec![0; INPUT_BUFFER_SIZE]; + + assert!(!input_buffer.is_empty()); + + // Data that was read from `input` but has not been encoded yet + let mut leftover_buffer = VecDeque::::new(); + + // Encoded data that needs to be written to `output` + let mut encoded_buffer = VecDeque::::new(); + // End of buffers + + loop { + match input.read(&mut input_buffer) { + Ok(bytes_read_from_input) => { + if bytes_read_from_input == 0 { + break; + } + + // The part of `input_buffer` that was actually filled by the call to `read` + let read_buffer = &input_buffer[..bytes_read_from_input]; + + // How many bytes to steal from `read_buffer` to get `leftover_buffer` to the right size + let bytes_to_steal = encode_in_chunks_of_size - leftover_buffer.len(); + + if bytes_to_steal > bytes_read_from_input { + // Do not have enough data to encode a chunk, so copy data to `leftover_buffer` and read more + leftover_buffer.extend(read_buffer); + + assert!(leftover_buffer.len() < encode_in_chunks_of_size); + + continue; + } + + // Encode data in chunks, then place it in `encoded_buffer` + encode_in_chunks_to_buffer( + supports_fast_decode_and_encode, + encode_in_chunks_of_size, + bytes_to_steal, + read_buffer, + &mut encoded_buffer, + &mut leftover_buffer, + )?; + + assert!(leftover_buffer.len() < encode_in_chunks_of_size); + // Write all data in `encoded_buffer` to `output` + write_to_output( + &mut line_wrapping, + &mut encoded_buffer, + output, + false, + wrap == Some(0), + )?; + } + Err(er) => { + let kind = er.kind(); + + if kind == ErrorKind::Interrupted { + // Retry reading + continue; + } + + return Err(USimpleError::new(1, format_read_error(kind))); } - Ok(()) } - Err(_) => Err(USimpleError::new(1, "error: invalid input")), } - } else { - match data.encode() { - Ok(s) => { - wrap_print(&data, &s); - Ok(()) + + // Cleanup + // `input` has finished producing data, so the data remaining in the buffers needs to be encoded and printed + { + // Encode all remaining unencoded bytes, placing them in `encoded_buffer` + supports_fast_decode_and_encode + .encode_to_vec_deque(leftover_buffer.make_contiguous(), &mut encoded_buffer)?; + + // Write all data in `encoded_buffer` to output + // `is_cleanup` triggers special cleanup-only logic + write_to_output( + &mut line_wrapping, + &mut encoded_buffer, + output, + true, + wrap == Some(0), + )?; + } + + Ok(()) + } +} + +pub mod fast_decode { + use crate::base_common::format_read_error; + use std::io::{self, ErrorKind, Read, Write}; + use uucore::{ + encoding::SupportsFastDecodeAndEncode, + error::{UResult, USimpleError}, + }; + + // Start of helper functions + fn alphabet_to_table(alphabet: &[u8], ignore_garbage: bool) -> [bool; 256] { + // If `ignore_garbage` is enabled, all characters outside the alphabet are ignored + // If it is not enabled, only '\n' and '\r' are ignored + if ignore_garbage { + // Note: "false" here + let mut table = [false; 256]; + + // Pass through no characters except those in the alphabet + for ue in alphabet { + let us = usize::from(*ue); + + // Should not have been set yet + assert!(!table[us]); + + table[us] = true; + } + + table + } else { + // Note: "true" here + let mut table = [true; 256]; + + // Pass through all characters except '\n' and '\r' + for ue in [b'\n', b'\r'] { + let us = usize::from(ue); + + // Should not have been set yet + assert!(table[us]); + + table[us] = false; + } + + table + } + } + + fn decode_in_chunks_to_buffer( + supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode, + decode_in_chunks_of_size: usize, + bytes_to_steal: usize, + read_buffer_filtered: &[u8], + decoded_buffer: &mut Vec, + leftover_buffer: &mut Vec, + ) -> UResult<()> { + let bytes_to_chunk = if bytes_to_steal > 0 { + let (stolen_bytes, rest_of_read_buffer_filtered) = + read_buffer_filtered.split_at(bytes_to_steal); + + leftover_buffer.extend(stolen_bytes); + + // After appending the stolen bytes to `leftover_buffer`, it should be the right size + assert_eq!(leftover_buffer.len(), decode_in_chunks_of_size); + + // Decode the old un-decoded data and the stolen bytes, and add the result to + // `decoded_buffer` + supports_fast_decode_and_encode.decode_into_vec(leftover_buffer, decoded_buffer)?; + + // Reset `leftover_buffer` + leftover_buffer.clear(); + + rest_of_read_buffer_filtered + } else { + // Do not need to steal bytes from `read_buffer` + read_buffer_filtered + }; + + let chunks_exact = bytes_to_chunk.chunks_exact(decode_in_chunks_of_size); + + let remainder = chunks_exact.remainder(); + + for sl in chunks_exact { + assert_eq!(sl.len(), decode_in_chunks_of_size); + + supports_fast_decode_and_encode.decode_into_vec(sl, decoded_buffer)?; + } + + leftover_buffer.extend(remainder); + + Ok(()) + } + + fn write_to_output(decoded_buffer: &mut Vec, output: &mut dyn Write) -> io::Result<()> { + // Write all data in `decoded_buffer` to `output` + output.write_all(decoded_buffer.as_slice())?; + + decoded_buffer.clear(); + + Ok(()) + } + // End of helper functions + + pub fn fast_decode( + input: &mut dyn Read, + output: &mut dyn Write, + supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode, + ignore_garbage: bool, + ) -> UResult<()> { + // Based on performance testing + const INPUT_BUFFER_SIZE: usize = 32 * 1_024; + + const DECODE_IN_CHUNKS_OF_SIZE_MULTIPLE: usize = 1_024; + + let alphabet = supports_fast_decode_and_encode.alphabet(); + let decode_in_chunks_of_size = supports_fast_decode_and_encode.valid_decoding_multiple() + * DECODE_IN_CHUNKS_OF_SIZE_MULTIPLE; + + assert!(decode_in_chunks_of_size > 0); + + // Note that it's not worth using "data-encoding"'s ignore functionality if `ignore_garbage` is true, because + // "data-encoding"'s ignore functionality cannot discard non-ASCII bytes. The data has to be filtered before + // passing it to "data-encoding", so there is no point in doing any filtering in "data-encoding". This also + // allows execution to stay on the happy path in "data-encoding": + // https://github.com/ia0/data-encoding/blob/4f42ad7ef242f6d243e4de90cd1b46a57690d00e/lib/src/lib.rs#L754-L756 + // It is also not worth using "data-encoding"'s ignore functionality when `ignore_garbage` is + // false. + // Note that the alphabet constants above already include the padding characters + // TODO + // Precompute this + let table = alphabet_to_table(alphabet, ignore_garbage); + + // Start of buffers + // Data that was read from `input` + let mut input_buffer = vec![0; INPUT_BUFFER_SIZE]; + + assert!(!input_buffer.is_empty()); + + // Data that was read from `input` but has not been decoded yet + let mut leftover_buffer = Vec::::new(); + + // Decoded data that needs to be written to `output` + let mut decoded_buffer = Vec::::new(); + + // Buffer that will be used when `ignore_garbage` is true, and the chunk read from `input` contains garbage + // data + let mut non_garbage_buffer = Vec::::new(); + // End of buffers + + loop { + match input.read(&mut input_buffer) { + Ok(bytes_read_from_input) => { + if bytes_read_from_input == 0 { + break; + } + + let read_buffer_filtered = { + // The part of `input_buffer` that was actually filled by the call to `read` + let read_buffer = &input_buffer[..bytes_read_from_input]; + + // First just scan the data for the happy path + // Yields significant speedup when the input does not contain line endings + let found_garbage = read_buffer.iter().any(|ue| { + // Garbage, since it was not found in the table + !table[usize::from(*ue)] + }); + + if found_garbage { + non_garbage_buffer.clear(); + + for ue in read_buffer { + if table[usize::from(*ue)] { + // Not garbage, since it was found in the table + non_garbage_buffer.push(*ue); + } + } + + non_garbage_buffer.as_slice() + } else { + read_buffer + } + }; + + // How many bytes to steal from `read_buffer` to get `leftover_buffer` to the right size + let bytes_to_steal = decode_in_chunks_of_size - leftover_buffer.len(); + + if bytes_to_steal > read_buffer_filtered.len() { + // Do not have enough data to decode a chunk, so copy data to `leftover_buffer` and read more + leftover_buffer.extend(read_buffer_filtered); + + assert!(leftover_buffer.len() < decode_in_chunks_of_size); + + continue; + } + + // Decode data in chunks, then place it in `decoded_buffer` + decode_in_chunks_to_buffer( + supports_fast_decode_and_encode, + decode_in_chunks_of_size, + bytes_to_steal, + read_buffer_filtered, + &mut decoded_buffer, + &mut leftover_buffer, + )?; + + assert!(leftover_buffer.len() < decode_in_chunks_of_size); + + // Write all data in `decoded_buffer` to `output` + write_to_output(&mut decoded_buffer, output)?; + } + Err(er) => { + let kind = er.kind(); + + if kind == ErrorKind::Interrupted { + // Retry reading + continue; + } + + return Err(USimpleError::new(1, format_read_error(kind))); + } + } + } + + // Cleanup + // `input` has finished producing data, so the data remaining in the buffers needs to be decoded and printed + { + // Decode all remaining encoded bytes, placing them in `decoded_buffer` + supports_fast_decode_and_encode + .decode_into_vec(&leftover_buffer, &mut decoded_buffer)?; + + // Write all data in `decoded_buffer` to `output` + write_to_output(&mut decoded_buffer, output)?; + } + + Ok(()) + } +} + +fn format_read_error(kind: ErrorKind) -> String { + let kind_string = kind.to_string(); + + // e.g. "is a directory" -> "Is a directory" + let mut kind_string_capitalized = String::with_capacity(kind_string.len()); + + for (index, ch) in kind_string.char_indices() { + if index == 0 { + for cha in ch.to_uppercase() { + kind_string_capitalized.push(cha); } - Err(EncodeError::InvalidInput) => Err(USimpleError::new(1, "error: invalid input")), - Err(_) => Err(USimpleError::new( - 1, - "error: invalid input (length must be multiple of 4 characters)", - )), + } else { + kind_string_capitalized.push(ch); + } + } + + format!("read error: {kind_string_capitalized}") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn test_has_padding() { + let test_cases = vec![ + ("aGVsbG8sIHdvcmxkIQ==", true), + ("aGVsbG8sIHdvcmxkIQ== ", true), + ("aGVsbG8sIHdvcmxkIQ==\n", true), + ("aGVsbG8sIHdvcmxkIQ== \n", true), + ("aGVsbG8sIHdvcmxkIQ=", true), + ("aGVsbG8sIHdvcmxkIQ= ", true), + ("aGVsbG8sIHdvcmxkIQ \n", false), + ("aGVsbG8sIHdvcmxkIQ", false), + ]; + + for (input, expected) in test_cases { + let mut cursor = Cursor::new(input.as_bytes()); + assert_eq!( + has_padding(&mut cursor).unwrap(), + expected, + "Failed for input: '{input}'" + ); } } } diff --git a/src/uu/base64/Cargo.toml b/src/uu/base64/Cargo.toml index 204b880bf72..aa899f1a1e6 100644 --- a/src/uu/base64/Cargo.toml +++ b/src/uu/base64/Cargo.toml @@ -1,20 +1,24 @@ [package] name = "uu_base64" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "base64 ~ (uutils) decode/encode input (base64-encoding)" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/base64" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/base64.rs" [dependencies] +clap = { workspace = true } uucore = { workspace = true, features = ["encoding"] } uu_base32 = { workspace = true } diff --git a/src/uu/base64/src/base64.rs b/src/uu/base64/src/base64.rs index 6544638bdae..86eb75bf119 100644 --- a/src/uu/base64/src/base64.rs +++ b/src/uu/base64/src/base64.rs @@ -3,13 +3,10 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +use clap::Command; use uu_base32::base_common; -pub use uu_base32::uu_app; - use uucore::{encoding::Format, error::UResult, help_about, help_usage}; -use std::io::{stdin, Read}; - const ABOUT: &str = help_about!("base64.md"); const USAGE: &str = help_usage!("base64.md"); @@ -17,18 +14,13 @@ const USAGE: &str = help_usage!("base64.md"); pub fn uumain(args: impl uucore::Args) -> UResult<()> { let format = Format::Base64; - let config: base_common::Config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; + let config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; - // Create a reference to stdin so we can return a locked stdin from - // parse_base_cmd_args - let stdin_raw = stdin(); - let mut input: Box = base_common::get_input(&config, &stdin_raw)?; + let mut input = base_common::get_input(&config)?; + + base_common::handle_input(&mut input, format, config) +} - base_common::handle_input( - &mut input, - format, - config.wrap_cols, - config.ignore_garbage, - config.decode, - ) +pub fn uu_app() -> Command { + base_common::base_app(ABOUT, USAGE) } diff --git a/src/uu/basename/Cargo.toml b/src/uu/basename/Cargo.toml index 51202235b15..5123174ae2d 100644 --- a/src/uu/basename/Cargo.toml +++ b/src/uu/basename/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_basename" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "basename ~ (uutils) display PATHNAME with leading directory components removed" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/basename" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/basename.rs" diff --git a/src/uu/basename/basename.md b/src/uu/basename/basename.md index b17cac74a00..ee87fa76d4e 100644 --- a/src/uu/basename/basename.md +++ b/src/uu/basename/basename.md @@ -1,7 +1,7 @@ # basename ``` -basename NAME [SUFFIX] +basename [-z] NAME [SUFFIX] basename OPTION... NAME... ``` diff --git a/src/uu/basename/src/basename.rs b/src/uu/basename/src/basename.rs index 6c9baca6fce..a40fcc18534 100644 --- a/src/uu/basename/src/basename.rs +++ b/src/uu/basename/src/basename.rs @@ -5,8 +5,8 @@ // spell-checker:ignore (ToDO) fullname -use clap::{crate_version, Arg, ArgAction, Command}; -use std::path::{is_separator, PathBuf}; +use clap::{Arg, ArgAction, Command}; +use std::path::{PathBuf, is_separator}; use uucore::display::Quotable; use uucore::error::{UResult, UUsageError}; use uucore::line_ending::LineEnding; @@ -27,86 +27,48 @@ pub mod options { pub fn uumain(args: impl uucore::Args) -> UResult<()> { let args = args.collect_lossy(); - // Since options have to go before names, - // if the first argument is not an option, then there is no option, - // and that implies there is exactly one name (no option => no -a option), - // so simple format is used - if args.len() > 1 && !args[1].starts_with('-') { - if args.len() > 3 { - return Err(UUsageError::new( - 1, - format!("extra operand {}", args[3].to_string().quote()), - )); - } - let suffix = if args.len() > 2 { args[2].as_ref() } else { "" }; - println!("{}", basename(&args[1], suffix)); - return Ok(()); - } - // // Argument parsing // let matches = uu_app().try_get_matches_from(args)?; - // too few arguments - if !matches.contains_id(options::NAME) { - return Err(UUsageError::new(1, "missing operand".to_string())); - } - let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO)); - let opt_suffix = matches.get_one::(options::SUFFIX).is_some(); - let opt_multiple = matches.get_flag(options::MULTIPLE); - let multiple_paths = opt_suffix || opt_multiple; - let name_args_count = matches + let mut name_args = matches .get_many::(options::NAME) - .map(|n| n.len()) - .unwrap_or(0); - - // too many arguments - if !multiple_paths && name_args_count > 2 { - return Err(UUsageError::new( - 1, - format!( - "extra operand {}", - matches - .get_many::(options::NAME) - .unwrap() - .nth(2) - .unwrap() - .quote() - ), - )); + .unwrap_or_default() + .collect::>(); + if name_args.is_empty() { + return Err(UUsageError::new(1, "missing operand".to_string())); } - - let suffix = if opt_suffix { - matches.get_one::(options::SUFFIX).unwrap() - } else if !opt_multiple && name_args_count > 1 { + let multiple_paths = + matches.get_one::(options::SUFFIX).is_some() || matches.get_flag(options::MULTIPLE); + let suffix = if multiple_paths { matches - .get_many::(options::NAME) - .unwrap() - .nth(1) - .unwrap() + .get_one::(options::SUFFIX) + .cloned() + .unwrap_or_default() } else { - "" + // "simple format" + match name_args.len() { + 0 => panic!("already checked"), + 1 => String::default(), + 2 => name_args.pop().unwrap().clone(), + _ => { + return Err(UUsageError::new( + 1, + format!("extra operand {}", name_args[2].quote()), + )); + } + } }; // // Main Program Processing // - let paths: Vec<_> = if multiple_paths { - matches.get_many::(options::NAME).unwrap().collect() - } else { - matches - .get_many::(options::NAME) - .unwrap() - .take(1) - .collect() - }; - - for path in paths { - print!("{}{}", basename(path, suffix), line_ending); + for path in name_args { + print!("{}{line_ending}", basename(path, &suffix)); } Ok(()) @@ -114,7 +76,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -123,27 +85,31 @@ pub fn uu_app() -> Command { .short('a') .long(options::MULTIPLE) .help("support multiple arguments and treat each as a NAME") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with(options::MULTIPLE), ) .arg( Arg::new(options::NAME) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) .value_hint(clap::ValueHint::AnyPath) - .hide(true), + .hide(true) + .trailing_var_arg(true), ) .arg( Arg::new(options::SUFFIX) .short('s') .long(options::SUFFIX) .value_name("SUFFIX") - .help("remove a trailing SUFFIX; implies -a"), + .help("remove a trailing SUFFIX; implies -a") + .overrides_with(options::SUFFIX), ) .arg( Arg::new(options::ZERO) .short('z') .long(options::ZERO) .help("end each output line with NUL, not newline") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with(options::ZERO), ) } @@ -159,16 +125,13 @@ fn basename(fullname: &str, suffix: &str) -> String { // Convert to path buffer and get last path component let pb = PathBuf::from(path); - match pb.components().last() { - Some(c) => { - let name = c.as_os_str().to_str().unwrap(); - if name == suffix { - name.to_string() - } else { - name.strip_suffix(suffix).unwrap_or(name).to_string() - } - } - None => String::new(), - } + pb.components().next_back().map_or_else(String::new, |c| { + let name = c.as_os_str().to_str().unwrap(); + if name == suffix { + name.to_string() + } else { + name.strip_suffix(suffix).unwrap_or(name).to_string() + } + }) } diff --git a/src/uu/basenc/BENCHMARKING.md b/src/uu/basenc/BENCHMARKING.md new file mode 100644 index 00000000000..8248cbbc53b --- /dev/null +++ b/src/uu/basenc/BENCHMARKING.md @@ -0,0 +1,177 @@ + + +# Benchmarking base32, base64, and basenc + +Note that the functionality of the `base32` and `base64` programs is identical to that of the `basenc` program, using +the "--base32" and "--base64" options, respectively. For that reason, it is only necessary to benchmark `basenc`. + +To compare the runtime performance of the uutils implementation with the GNU Core Utilities implementation, you can +use a benchmarking tool like [hyperfine][0]. + +hyperfine currently does not measure maximum memory usage. Memory usage can be benchmarked using [poop][2], or +[toybox][3]'s "time" subcommand (both are Linux only). + +Build the `basenc` binary using the release profile: + +```Shell +cargo build --package uu_basenc --profile release +``` + +## Expected performance + +uutils' `basenc` performs streaming decoding and encoding, and therefore should perform all operations with a constant +maximum memory usage, regardless of the size of the input. Release builds currently use less than 3 mebibytes of +memory, and memory usage greater than 10 mebibytes should be considered a bug. + +As of September 2024, uutils' `basenc` has runtime performance equal to or superior to GNU Core Utilities' `basenc` in +in most scenarios. uutils' `basenc` uses slightly more memory, but given how small these quantities are in absolute +terms (see above), this is highly unlikely to be practically relevant to users. + +## Benchmark results (2024-09-27) + +### Setup + +```Shell +# Use uutils' dd to create a 1 gibibyte in-memory file filled with random bytes (Linux only). +# On other platforms, you can use /tmp instead of /dev/shm, but note that /tmp is not guaranteed to be in-memory. +coreutils dd if=/dev/urandom of=/dev/shm/one-random-gibibyte bs=1024 count=1048576 + +# Encode this file for use in decoding performance testing +/usr/bin/basenc --base32hex -- /dev/shm/one-random-gibibyte 1>/dev/shm/one-random-gibibyte-base32hex-encoded +/usr/bin/basenc --z85 -- /dev/shm/one-random-gibibyte 1>/dev/shm/one-random-gibibyte-z85-encoded +``` + +### Programs being tested + +uutils' `basenc`: + +``` +⯠git rev-list HEAD | coreutils head -n 1 -- - +a0718ef0ffd50539a2e2bc0095c9fadcd70ab857 +``` + +GNU Core Utilities' `basenc`: + +``` +⯠/usr/bin/basenc --version | coreutils head -n 1 -- - +basenc (GNU coreutils) 9.4 +``` + +### Encoding performance + +#### "--base64", default line wrapping (76 characters) + +➕ Faster than GNU Core Utilities + +``` +⯠hyperfine \ + --sort \ + command \ + -- \ + '/usr/bin/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null' \ + './target/release/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null' + +Benchmark 1: /usr/bin/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null + Time (mean ± σ): 965.1 ms ± 7.9 ms [User: 766.2 ms, System: 193.4 ms] + Range (min … max): 950.2 ms … 976.9 ms 10 runs + +Benchmark 2: ./target/release/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null + Time (mean ± σ): 696.6 ms ± 9.1 ms [User: 574.9 ms, System: 117.3 ms] + Range (min … max): 683.1 ms … 713.5 ms 10 runs + +Relative speed comparison + 1.39 ± 0.02 /usr/bin/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null + 1.00 ./target/release/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null +``` + +#### "--base16", no line wrapping + +➖ Slower than GNU Core Utilities + +``` +⯠poop \ + '/usr/bin/basenc --base16 --wrap 0 -- /dev/shm/one-random-gibibyte' \ + './target/release/basenc --base16 --wrap 0 -- /dev/shm/one-random-gibibyte' + +Benchmark 1 (6 runs): /usr/bin/basenc --base16 --wrap 0 -- /dev/shm/one-random-gibibyte + measurement mean ± σ min … max outliers delta + wall_time 836ms ± 13.3ms 822ms … 855ms 0 ( 0%) 0% + peak_rss 2.05MB ± 73.0KB 1.94MB … 2.12MB 0 ( 0%) 0% + cpu_cycles 2.85G ± 32.8M 2.82G … 2.91G 0 ( 0%) 0% + instructions 14.0G ± 58.7 14.0G … 14.0G 0 ( 0%) 0% + cache_references 70.0M ± 6.48M 63.7M … 78.8M 0 ( 0%) 0% + cache_misses 582K ± 172K 354K … 771K 0 ( 0%) 0% + branch_misses 667K ± 4.55K 662K … 674K 0 ( 0%) 0% +Benchmark 2 (6 runs): ./target/release/basenc --base16 --wrap 0 -- /dev/shm/one-random-gibibyte + measurement mean ± σ min … max outliers delta + wall_time 884ms ± 6.38ms 878ms … 895ms 0 ( 0%) 💩+ 5.7% ± 1.6% + peak_rss 2.65MB ± 66.8KB 2.55MB … 2.74MB 0 ( 0%) 💩+ 29.3% ± 4.4% + cpu_cycles 3.15G ± 8.61M 3.14G … 3.16G 0 ( 0%) 💩+ 10.6% ± 1.1% + instructions 10.5G ± 275 10.5G … 10.5G 0 ( 0%) ⚡- 24.9% ± 0.0% + cache_references 93.5M ± 6.10M 87.2M … 104M 0 ( 0%) 💩+ 33.7% ± 11.6% + cache_misses 415K ± 52.3K 363K … 474K 0 ( 0%) - 28.8% ± 28.0% + branch_misses 1.43M ± 4.82K 1.42M … 1.43M 0 ( 0%) 💩+113.9% ± 0.9% +``` + +### Decoding performance + +#### "--base32hex" + +➕ Faster than GNU Core Utilities + +``` +⯠hyperfine \ + --sort \ + command \ + -- \ + '/usr/bin/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null' \ + './target/release/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null' + +Benchmark 1: /usr/bin/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null + Time (mean ± σ): 7.154 s ± 0.082 s [User: 6.802 s, System: 0.323 s] + Range (min … max): 7.051 s … 7.297 s 10 runs + +Benchmark 2: ./target/release/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null + Time (mean ± σ): 2.679 s ± 0.025 s [User: 2.446 s, System: 0.221 s] + Range (min … max): 2.649 s … 2.718 s 10 runs + +Relative speed comparison + 2.67 ± 0.04 /usr/bin/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null + 1.00 ./target/release/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null +``` + +#### "--z85", with "--ignore-garbage" + +➕ Faster than GNU Core Utilities + +``` +⯠poop \ + '/usr/bin/basenc --decode --ignore-garbage --z85 -- /dev/shm/one-random-gibibyte-z85-encoded' \ + './target/release/basenc --decode --ignore-garbage --z85 -- /dev/shm/one-random-gibibyte-z85-encoded' + +Benchmark 1 (3 runs): /usr/bin/basenc --decode --ignore-garbage --z85 -- /dev/shm/one-random-gibibyte-z85-encoded + measurement mean ± σ min … max outliers delta + wall_time 14.4s ± 68.4ms 14.3s … 14.4s 0 ( 0%) 0% + peak_rss 1.98MB ± 10.8KB 1.97MB … 1.99MB 0 ( 0%) 0% + cpu_cycles 58.4G ± 211M 58.3G … 58.7G 0 ( 0%) 0% + instructions 74.7G ± 64.0 74.7G … 74.7G 0 ( 0%) 0% + cache_references 41.8M ± 624K 41.2M … 42.4M 0 ( 0%) 0% + cache_misses 693K ± 118K 567K … 802K 0 ( 0%) 0% + branch_misses 1.24G ± 183K 1.24G … 1.24G 0 ( 0%) 0% +Benchmark 2 (3 runs): ./target/release/basenc --decode --ignore-garbage --z85 -- /dev/shm/one-random-gibibyte-z85-encoded + measurement mean ± σ min … max outliers delta + wall_time 2.80s ± 17.9ms 2.79s … 2.82s 0 ( 0%) ⚡- 80.5% ± 0.8% + peak_rss 2.61MB ± 67.4KB 2.57MB … 2.69MB 0 ( 0%) 💩+ 31.9% ± 5.5% + cpu_cycles 10.8G ± 27.9M 10.8G … 10.9G 0 ( 0%) ⚡- 81.5% ± 0.6% + instructions 39.0G ± 353 39.0G … 39.0G 0 ( 0%) ⚡- 47.7% ± 0.0% + cache_references 114M ± 2.43M 112M … 116M 0 ( 0%) 💩+173.3% ± 9.6% + cache_misses 1.06M ± 288K 805K … 1.37M 0 ( 0%) + 52.6% ± 72.0% + branch_misses 1.18M ± 14.7K 1.16M … 1.19M 0 ( 0%) ⚡- 99.9% ± 0.0% +``` + +[0]: https://github.com/sharkdp/hyperfine +[1]: https://github.com/sharkdp/hyperfine?tab=readme-ov-file#installation +[2]: https://github.com/andrewrk/poop +[3]: https://landley.net/toybox/ diff --git a/src/uu/basenc/Cargo.toml b/src/uu/basenc/Cargo.toml index 26a2364282f..2f78a95751c 100644 --- a/src/uu/basenc/Cargo.toml +++ b/src/uu/basenc/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_basenc" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "basenc ~ (uutils) decode/encode input" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/basenc" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/basenc.rs" diff --git a/src/uu/basenc/src/basenc.rs b/src/uu/basenc/src/basenc.rs index ff512b17652..10090765232 100644 --- a/src/uu/basenc/src/basenc.rs +++ b/src/uu/basenc/src/basenc.rs @@ -3,19 +3,15 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -//spell-checker:ignore (args) lsbf msbf +// spell-checker:ignore lsbf msbf use clap::{Arg, ArgAction, Command}; -use uu_base32::base_common::{self, Config, BASE_CMD_PARSE_ERROR}; - +use uu_base32::base_common::{self, BASE_CMD_PARSE_ERROR, Config}; +use uucore::error::UClapError; use uucore::{ encoding::Format, error::{UResult, UUsageError}, }; - -use std::io::{stdin, Read}; -use uucore::error::UClapError; - use uucore::{help_about, help_usage}; const ABOUT: &str = help_about!("basenc.md"); @@ -53,12 +49,14 @@ const ENCODINGS: &[(&str, Format, &str)] = &[ pub fn uu_app() -> Command { let mut command = base_common::base_app(ABOUT, USAGE); for encoding in ENCODINGS { - command = command.arg( - Arg::new(encoding.0) - .long(encoding.0) - .help(encoding.2) - .action(ArgAction::SetTrue), - ); + let raw_arg = Arg::new(encoding.0) + .long(encoding.0) + .help(encoding.2) + .action(ArgAction::SetTrue); + let overriding_arg = ENCODINGS + .iter() + .fold(raw_arg, |arg, enc| arg.overrides_with(enc.0)); + command = command.arg(overriding_arg); } command } @@ -79,16 +77,8 @@ fn parse_cmd_args(args: impl uucore::Args) -> UResult<(Config, Format)> { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let (config, format) = parse_cmd_args(args)?; - // Create a reference to stdin so we can return a locked stdin from - // parse_base_cmd_args - let stdin_raw = stdin(); - let mut input: Box = base_common::get_input(&config, &stdin_raw)?; - base_common::handle_input( - &mut input, - format, - config.wrap_cols, - config.ignore_garbage, - config.decode, - ) + let mut input = base_common::get_input(&config)?; + + base_common::handle_input(&mut input, format, config) } diff --git a/src/uu/cat/Cargo.toml b/src/uu/cat/Cargo.toml index cce6561c084..f5ac6a64eff 100644 --- a/src/uu/cat/Cargo.toml +++ b/src/uu/cat/Cargo.toml @@ -1,23 +1,27 @@ [package] name = "uu_cat" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "cat ~ (uutils) concatenate and display input" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/cat" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/cat.rs" [dependencies] clap = { workspace = true } +memchr = { workspace = true } thiserror = { workspace = true } -uucore = { workspace = true, features = ["fs", "pipes"] } +uucore = { workspace = true, features = ["fast-inc", "fs", "pipes"] } [target.'cfg(unix)'.dependencies] nix = { workspace = true } diff --git a/src/uu/cat/src/cat.rs b/src/uu/cat/src/cat.rs index af55442ca5e..c0a41270f34 100644 --- a/src/uu/cat/src/cat.rs +++ b/src/uu/cat/src/cat.rs @@ -4,33 +4,87 @@ // file that was distributed with this source code. // spell-checker:ignore (ToDO) nonprint nonblank nonprinting ELOOP -use clap::{crate_version, Arg, ArgAction, Command}; -use std::fs::{metadata, File}; -use std::io::{self, IsTerminal, Read, Write}; +use std::fs::{File, metadata}; +use std::io::{self, BufWriter, IsTerminal, Read, Write}; +/// Unix domain socket support +#[cfg(unix)] +use std::net::Shutdown; +#[cfg(unix)] +use std::os::fd::{AsFd, AsRawFd}; +#[cfg(unix)] +use std::os::unix::fs::FileTypeExt; +#[cfg(unix)] +use std::os::unix::net::UnixStream; + +use clap::{Arg, ArgAction, Command}; +use memchr::memchr2; +#[cfg(unix)] +use nix::fcntl::{FcntlArg, fcntl}; use thiserror::Error; use uucore::display::Quotable; use uucore::error::UResult; use uucore::fs::FileInformation; - -#[cfg(unix)] -use std::os::unix::io::AsRawFd; +use uucore::{fast_inc::fast_inc_one, format_usage, help_about, help_usage}; /// Linux splice support #[cfg(any(target_os = "linux", target_os = "android"))] mod splice; -/// Unix domain socket support -#[cfg(unix)] -use std::net::Shutdown; -#[cfg(unix)] -use std::os::unix::fs::FileTypeExt; -#[cfg(unix)] -use std::os::unix::net::UnixStream; -use uucore::{format_usage, help_about, help_usage}; - const USAGE: &str = help_usage!("cat.md"); const ABOUT: &str = help_about!("cat.md"); +// Allocate 32 digits for the line number. +// An estimate is that we can print about 1e8 lines/seconds, so 32 digits +// would be enough for billions of universe lifetimes. +const LINE_NUMBER_BUF_SIZE: usize = 32; + +struct LineNumber { + buf: [u8; LINE_NUMBER_BUF_SIZE], + print_start: usize, + num_start: usize, + num_end: usize, +} + +// Logic to store a string for the line number. Manually incrementing the value +// represented in a buffer like this is significantly faster than storing +// a `usize` and using the standard Rust formatting macros to format a `usize` +// to a string each time it's needed. +// Buffer is initialized to " 1\t" and incremented each time `increment` is +// called, using uucore's fast_inc function that operates on strings. +impl LineNumber { + fn new() -> Self { + let mut buf = [b'0'; LINE_NUMBER_BUF_SIZE]; + + let init_str = " 1\t"; + let print_start = buf.len() - init_str.len(); + let num_start = buf.len() - 2; + let num_end = buf.len() - 1; + + buf[print_start..].copy_from_slice(init_str.as_bytes()); + + LineNumber { + buf, + print_start, + num_start, + num_end, + } + } + + fn increment(&mut self) { + fast_inc_one(&mut self.buf, &mut self.num_start, self.num_end); + self.print_start = self.print_start.min(self.num_start); + } + + #[inline] + fn to_str(&self) -> &[u8] { + &self.buf[self.print_start..] + } + + fn write(&self, writer: &mut impl Write) -> io::Result<()> { + writer.write_all(self.to_str()) + } +} + #[derive(Error, Debug)] enum CatError { /// Wrapper around `io::Error` @@ -41,7 +95,7 @@ enum CatError { #[error("{0}")] Nix(#[from] nix::Error), /// Unknown file type; it's not a regular file, socket, etc. - #[error("unknown filetype: {}", ft_debug)] + #[error("unknown filetype: {ft_debug}")] UnknownFiletype { /// A debug print of the file type ft_debug: String, @@ -82,19 +136,11 @@ struct OutputOptions { impl OutputOptions { fn tab(&self) -> &'static str { - if self.show_tabs { - "^I" - } else { - "\t" - } + if self.show_tabs { "^I" } else { "\t" } } fn end_of_line(&self) -> &'static str { - if self.show_ends { - "$\n" - } else { - "\n" - } + if self.show_ends { "$\n" } else { "\n" } } /// We can write fast if we can simply copy the contents of the file to @@ -112,7 +158,7 @@ impl OutputOptions { /// when we can't write fast. struct OutputState { /// The current line number - line_number: usize, + line_number: LineNumber, /// Whether the output cursor is at the beginning of a new line at_line_start: bool, @@ -125,12 +171,12 @@ struct OutputState { } #[cfg(unix)] -trait FdReadable: Read + AsRawFd {} +trait FdReadable: Read + AsFd {} #[cfg(not(unix))] trait FdReadable: Read {} #[cfg(unix)] -impl FdReadable for T where T: Read + AsRawFd {} +impl FdReadable for T where T: Read + AsFd {} #[cfg(not(unix))] impl FdReadable for T where T: Read {} @@ -170,6 +216,7 @@ mod options { pub static SHOW_NONPRINTING_TABS: &str = "t"; pub static SHOW_TABS: &str = "show-tabs"; pub static SHOW_NONPRINTING: &str = "show-nonprinting"; + pub static IGNORED_U: &str = "ignored-u"; } #[uucore::main] @@ -227,14 +274,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .override_usage(format_usage(USAGE)) .about(ABOUT) .infer_long_args(true) + .args_override_self(true) .arg( Arg::new(options::FILE) .hide(true) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) .value_hint(clap::ValueHint::FilePath), ) .arg( @@ -249,7 +297,8 @@ pub fn uu_app() -> Command { .short('b') .long(options::NUMBER_NONBLANK) .help("number nonempty output lines, overrides -n") - .overrides_with(options::NUMBER) + // Note: This MUST NOT .overrides_with(options::NUMBER)! + // In clap, overriding is symmetric, so "-b -n" counts as "-n", which is not what we want. .action(ArgAction::SetTrue), ) .arg( @@ -299,6 +348,12 @@ pub fn uu_app() -> Command { .help("use ^ and M- notation, except for LF (\\n) and TAB (\\t)") .action(ArgAction::SetTrue), ) + .arg( + Arg::new(options::IGNORED_U) + .short('u') + .help("(ignored)") + .action(ArgAction::SetTrue), + ) } fn cat_handle( @@ -313,6 +368,23 @@ fn cat_handle( } } +/// Whether this process is appending to stdout. +#[cfg(unix)] +fn is_appending() -> bool { + let stdout = io::stdout(); + let Ok(flags) = fcntl(stdout.as_raw_fd(), FcntlArg::F_GETFL) else { + return false; + }; + // TODO Replace `1 << 10` with `nix::fcntl::Oflag::O_APPEND`. + let o_append = 1 << 10; + (flags & o_append) > 0 +} + +#[cfg(not(unix))] +fn is_appending() -> bool { + false +} + fn cat_path( path: &str, options: &OutputOptions, @@ -322,10 +394,16 @@ fn cat_path( match get_input_type(path)? { InputType::StdIn => { let stdin = io::stdin(); + let in_info = FileInformation::from_file(&stdin)?; let mut handle = InputHandle { reader: stdin, - is_interactive: std::io::stdin().is_terminal(), + is_interactive: io::stdin().is_terminal(), }; + if let Some(out_info) = out_info { + if in_info == *out_info && is_appending() { + return Err(CatError::OutputIsInput); + } + } cat_handle(&mut handle, options, state) } InputType::Directory => Err(CatError::IsDirectory), @@ -360,10 +438,10 @@ fn cat_path( } fn cat_files(files: &[String], options: &OutputOptions) -> UResult<()> { - let out_info = FileInformation::from_file(&std::io::stdout()).ok(); + let out_info = FileInformation::from_file(&io::stdout()).ok(); let mut state = OutputState { - line_number: 1, + line_number: LineNumber::new(), at_line_start: true, skipped_carriage_return: false, one_blank_kept: false, @@ -372,7 +450,7 @@ fn cat_files(files: &[String], options: &OutputOptions) -> UResult<()> { for path in files { if let Err(err) = cat_path(path, options, &mut state, out_info.as_ref()) { - error_messages.push(format!("{}: {}", path.maybe_quote(), err)); + error_messages.push(format!("{}: {err}", path.maybe_quote())); } } if state.skipped_carriage_return { @@ -458,6 +536,13 @@ fn write_fast(handle: &mut InputHandle) -> CatResult<()> { } stdout_lock.write_all(&buf[..n])?; } + + // If the splice() call failed and there has been some data written to + // stdout via while loop above AND there will be second splice() call + // that will succeed, data pushed through splice will be output before + // the data buffered in stdout.lock. Therefore additional explicit flush + // is required here. + stdout_lock.flush()?; Ok(()) } @@ -470,7 +555,9 @@ fn write_lines( ) -> CatResult<()> { let mut in_buf = [0; 1024 * 31]; let stdout = io::stdout(); - let mut writer = stdout.lock(); + let stdout = stdout.lock(); + // Add a 32K buffer for stdout - this greatly improves performance. + let mut writer = BufWriter::with_capacity(32 * 1024, stdout); while let Ok(n) = handle.reader.read(&mut in_buf) { if n == 0 { @@ -493,8 +580,8 @@ fn write_lines( } state.one_blank_kept = false; if state.at_line_start && options.number != NumberingMode::None { - write!(writer, "{0:6}\t", state.line_number)?; - state.line_number += 1; + state.line_number.write(&mut writer)?; + state.line_number.increment(); } // print to end of line or end of buffer @@ -519,6 +606,14 @@ fn write_lines( } pos += offset + 1; } + // We need to flush the buffer each time around the loop in order to pass GNU tests. + // When we are reading the input from a pipe, the `handle.reader.read` call at the top + // of this loop will block (indefinitely) whist waiting for more data. The expectation + // however is that anything that's ready for output should show up in the meantime, + // and not be buffered internally to the `cat` process. + // Hence it's necessary to flush our buffer before every time we could potentially block + // on a `std::io::Read::read` call. + writer.flush()?; } Ok(()) @@ -531,20 +626,24 @@ fn write_new_line( state: &mut OutputState, is_interactive: bool, ) -> CatResult<()> { - if state.skipped_carriage_return && options.show_ends { - writer.write_all(b"^M")?; + if state.skipped_carriage_return { + if options.show_ends { + writer.write_all(b"^M")?; + } else { + writer.write_all(b"\r")?; + } state.skipped_carriage_return = false; + + write_end_of_line(writer, options.end_of_line().as_bytes(), is_interactive)?; + return Ok(()); } if !state.at_line_start || !options.squeeze_blank || !state.one_blank_kept { state.one_blank_kept = true; if state.at_line_start && options.number == NumberingMode::All { - write!(writer, "{0:6}\t", state.line_number)?; - state.line_number += 1; - } - writer.write_all(options.end_of_line().as_bytes())?; - if is_interactive { - writer.flush()?; + state.line_number.write(writer)?; + state.line_number.increment(); } + write_end_of_line(writer, options.end_of_line().as_bytes(), is_interactive)?; } Ok(()) } @@ -565,7 +664,8 @@ fn write_end(writer: &mut W, in_buf: &[u8], options: &OutputOptions) - // however, write_nonprint_to_end doesn't need to stop at \r because it will always write \r as ^M. // Return the number of written symbols fn write_to_end(in_buf: &[u8], writer: &mut W) -> usize { - match in_buf.iter().position(|c| *c == b'\n' || *c == b'\r') { + // using memchr2 significantly improves performances + match memchr2(b'\n', b'\r', in_buf) { Some(p) => { writer.write_all(&in_buf[..p]).unwrap(); p @@ -597,7 +697,7 @@ fn write_tab_to_end(mut in_buf: &[u8], writer: &mut W) -> usize { } None => { writer.write_all(in_buf).unwrap(); - return in_buf.len(); + return in_buf.len() + count; } }; } @@ -614,10 +714,10 @@ fn write_nonprint_to_end(in_buf: &[u8], writer: &mut W, tab: &[u8]) -> 9 => writer.write_all(tab), 0..=8 | 10..=31 => writer.write_all(&[b'^', byte + 64]), 32..=126 => writer.write_all(&[byte]), - 127 => writer.write_all(&[b'^', b'?']), + 127 => writer.write_all(b"^?"), 128..=159 => writer.write_all(&[b'M', b'-', b'^', byte - 64]), 160..=254 => writer.write_all(&[b'M', b'-', byte - 128]), - _ => writer.write_all(&[b'M', b'-', b'^', b'?']), + _ => writer.write_all(b"M-^?"), } .unwrap(); count += 1; @@ -639,7 +739,21 @@ fn write_end_of_line( #[cfg(test)] mod tests { - use std::io::{stdout, BufWriter}; + use std::io::{BufWriter, stdout}; + + #[test] + fn test_write_tab_to_end_with_newline() { + let mut writer = BufWriter::with_capacity(1024 * 64, stdout()); + let in_buf = b"a\tb\tc\n"; + assert_eq!(super::write_tab_to_end(in_buf, &mut writer), 5); + } + + #[test] + fn test_write_tab_to_end_no_newline() { + let mut writer = BufWriter::with_capacity(1024 * 64, stdout()); + let in_buf = b"a\tb\tc"; + assert_eq!(super::write_tab_to_end(in_buf, &mut writer), 5); + } #[test] fn test_write_nonprint_to_end_new_line() { @@ -680,4 +794,25 @@ mod tests { assert_eq!(writer.buffer(), [b'^', byte + 64]); } } + + #[test] + fn test_incrementing_string() { + let mut incrementing_string = super::LineNumber::new(); + assert_eq!(b" 1\t", incrementing_string.to_str()); + incrementing_string.increment(); + assert_eq!(b" 2\t", incrementing_string.to_str()); + // Run through to 100 + for _ in 3..=100 { + incrementing_string.increment(); + } + assert_eq!(b" 100\t", incrementing_string.to_str()); + // Run through until we overflow the original size. + for _ in 101..=1_000_000 { + incrementing_string.increment(); + } + // Confirm that the start position moves when we overflow the original size. + assert_eq!(b"1000000\t", incrementing_string.to_str()); + incrementing_string.increment(); + assert_eq!(b"1000001\t", incrementing_string.to_str()); + } } diff --git a/src/uu/cat/src/splice.rs b/src/uu/cat/src/splice.rs index 6c2b6d3dac3..13daae84d7f 100644 --- a/src/uu/cat/src/splice.rs +++ b/src/uu/cat/src/splice.rs @@ -5,7 +5,10 @@ use super::{CatResult, FdReadable, InputHandle}; use nix::unistd; -use std::os::unix::io::{AsRawFd, RawFd}; +use std::os::{ + fd::AsFd, + unix::io::{AsRawFd, RawFd}, +}; use uucore::pipes::{pipe, splice, splice_exact}; @@ -20,9 +23,9 @@ const BUF_SIZE: usize = 1024 * 16; /// The `bool` in the result value indicates if we need to fall back to normal /// copying or not. False means we don't have to. #[inline] -pub(super) fn write_fast_using_splice( +pub(super) fn write_fast_using_splice( handle: &InputHandle, - write_fd: &impl AsRawFd, + write_fd: &S, ) -> CatResult { let (pipe_rd, pipe_wr) = pipe()?; @@ -38,7 +41,7 @@ pub(super) fn write_fast_using_splice( // we can recover by copying the data that we have from the // intermediate pipe to stdout using normal read/write. Then // we tell the caller to fall back. - copy_exact(pipe_rd.as_raw_fd(), write_fd.as_raw_fd(), n)?; + copy_exact(pipe_rd.as_raw_fd(), write_fd, n)?; return Ok(true); } } @@ -52,7 +55,7 @@ pub(super) fn write_fast_using_splice( /// Move exactly `num_bytes` bytes from `read_fd` to `write_fd`. /// /// Panics if not enough bytes can be read. -fn copy_exact(read_fd: RawFd, write_fd: RawFd, num_bytes: usize) -> nix::Result<()> { +fn copy_exact(read_fd: RawFd, write_fd: &impl AsFd, num_bytes: usize) -> nix::Result<()> { let mut left = num_bytes; let mut buf = [0; BUF_SIZE]; while left > 0 { diff --git a/src/uu/chcon/Cargo.toml b/src/uu/chcon/Cargo.toml index 021e435823a..ccf36056339 100644 --- a/src/uu/chcon/Cargo.toml +++ b/src/uu/chcon/Cargo.toml @@ -1,14 +1,18 @@ [package] name = "uu_chcon" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "chcon ~ (uutils) change file security context" -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chcon" keywords = ["coreutils", "uutils", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chcon.rs" diff --git a/src/uu/chcon/src/chcon.rs b/src/uu/chcon/src/chcon.rs index ec111c853fd..2b1ff2e8f97 100644 --- a/src/uu/chcon/src/chcon.rs +++ b/src/uu/chcon/src/chcon.rs @@ -3,13 +3,14 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. // spell-checker:ignore (vars) RFILE +#![cfg(target_os = "linux")] #![allow(clippy::upper_case_acronyms)] use clap::builder::ValueParser; use uucore::error::{UResult, USimpleError, UUsageError}; use uucore::{display::Quotable, format_usage, help_about, help_usage, show_error, show_warning}; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use selinux::{OpaqueSecurityContext, SecurityContext}; use std::borrow::Cow; @@ -149,11 +150,12 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) .disable_help_flag(true) + .args_override_self(true) .arg( Arg::new(options::HELP) .long(options::HELP) @@ -163,7 +165,7 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::dereference::DEREFERENCE) .long(options::dereference::DEREFERENCE) - .conflicts_with(options::dereference::NO_DEREFERENCE) + .overrides_with(options::dereference::NO_DEREFERENCE) .help( "Affect the referent of each symbolic link (this is the default), \ rather than the symbolic link itself.", @@ -180,7 +182,7 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::preserve_root::PRESERVE_ROOT) .long(options::preserve_root::PRESERVE_ROOT) - .conflicts_with(options::preserve_root::NO_PRESERVE_ROOT) + .overrides_with(options::preserve_root::NO_PRESERVE_ROOT) .help("Fail to operate recursively on '/'.") .action(ArgAction::SetTrue), ) @@ -310,7 +312,7 @@ struct Options { files: Vec, } -fn parse_command_line(config: clap::Command, args: impl uucore::Args) -> Result { +fn parse_command_line(config: Command, args: impl uucore::Args) -> Result { let matches = config.try_get_matches_from(args)?; let verbose = matches.get_flag(options::VERBOSE); @@ -606,7 +608,7 @@ fn process_file( if result.is_ok() { if options.verbose { println!( - "{}: Changing security context of: {}", + "{}: changing security context of {}", uucore::util_name(), file_full_name.quote() ); @@ -726,7 +728,7 @@ fn get_root_dev_ino() -> Result { } fn root_dev_ino_check(root_dev_ino: Option, dir_dev_ino: DeviceAndINode) -> bool { - root_dev_ino.map_or(false, |root_dev_ino| root_dev_ino == dir_dev_ino) + root_dev_ino == Some(dir_dev_ino) } fn root_dev_ino_warn(dir_name: &Path) { @@ -776,7 +778,7 @@ enum SELinuxSecurityContext<'t> { String(Option), } -impl<'t> SELinuxSecurityContext<'t> { +impl SELinuxSecurityContext<'_> { fn to_c_string(&self) -> Result>> { match self { Self::File(context) => context diff --git a/src/uu/chcon/src/errors.rs b/src/uu/chcon/src/errors.rs index 10d5735a0c6..b8f720a3920 100644 --- a/src/uu/chcon/src/errors.rs +++ b/src/uu/chcon/src/errors.rs @@ -2,6 +2,8 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +#![cfg(target_os = "linux")] + use std::ffi::OsString; use std::fmt::Write; use std::io; diff --git a/src/uu/chcon/src/fts.rs b/src/uu/chcon/src/fts.rs index a81cb39b658..c9a8599fa2a 100644 --- a/src/uu/chcon/src/fts.rs +++ b/src/uu/chcon/src/fts.rs @@ -2,11 +2,12 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +#![cfg(target_os = "linux")] + use std::ffi::{CStr, CString, OsStr}; use std::marker::PhantomData; use std::os::raw::{c_int, c_long, c_short}; use std::path::Path; -use std::ptr::NonNull; use std::{io, iter, ptr, slice}; use crate::errors::{Error, Result}; @@ -69,7 +70,7 @@ impl FTS { // pointer assumed to be valid. let new_entry = unsafe { fts_sys::fts_read(self.fts.as_ptr()) }; - self.entry = NonNull::new(new_entry); + self.entry = ptr::NonNull::new(new_entry); if self.entry.is_none() { let r = io::Error::last_os_error(); if let Some(0) = r.raw_os_error() { @@ -159,7 +160,7 @@ impl<'fts> EntryRef<'fts> { return None; } - NonNull::new(entry.fts_path) + ptr::NonNull::new(entry.fts_path) .map(|path_ptr| { let path_size = usize::from(entry.fts_pathlen).saturating_add(1); diff --git a/src/uu/chcon/src/main.rs b/src/uu/chcon/src/main.rs index d93d7d1da2b..d1354d840af 100644 --- a/src/uu/chcon/src/main.rs +++ b/src/uu/chcon/src/main.rs @@ -1 +1,2 @@ +#![cfg(target_os = "linux")] uucore::bin!(uu_chcon); diff --git a/src/uu/chgrp/Cargo.toml b/src/uu/chgrp/Cargo.toml index bbad8d31e6e..7f23eec34d7 100644 --- a/src/uu/chgrp/Cargo.toml +++ b/src/uu/chgrp/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_chgrp" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "chgrp ~ (uutils) change the group ownership of FILE" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chgrp" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chgrp.rs" diff --git a/src/uu/chgrp/src/chgrp.rs b/src/uu/chgrp/src/chgrp.rs index fba2cef1611..1763bbfeb73 100644 --- a/src/uu/chgrp/src/chgrp.rs +++ b/src/uu/chgrp/src/chgrp.rs @@ -8,10 +8,10 @@ use uucore::display::Quotable; pub use uucore::entries; use uucore::error::{FromIo, UResult, USimpleError}; -use uucore::perms::{chown_base, options, GidUidOwnerFilter, IfFrom}; +use uucore::perms::{GidUidOwnerFilter, IfFrom, chown_base, options}; use uucore::{format_usage, help_about, help_usage}; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; use std::fs; use std::os::unix::fs::MetadataExt; @@ -19,8 +19,26 @@ use std::os::unix::fs::MetadataExt; const ABOUT: &str = help_about!("chgrp.md"); const USAGE: &str = help_usage!("chgrp.md"); -fn parse_gid_and_uid(matches: &ArgMatches) -> UResult { - let mut raw_group: String = String::new(); +fn parse_gid_from_str(group: &str) -> Result { + if let Some(gid_str) = group.strip_prefix(':') { + // Handle :gid format + gid_str + .parse::() + .map_err(|_| format!("invalid group id: '{gid_str}'")) + } else { + // Try as group name first + match entries::grp2gid(group) { + Ok(g) => Ok(g), + // If group name lookup fails, try parsing as raw number + Err(_) => group + .parse::() + .map_err(|_| format!("invalid group: '{group}'")), + } + } +} + +fn get_dest_gid(matches: &ArgMatches) -> UResult<(Option, String)> { + let mut raw_group = String::new(); let dest_gid = if let Some(file) = matches.get_one::(options::REFERENCE) { fs::metadata(file) .map(|meta| { @@ -38,22 +56,38 @@ fn parse_gid_and_uid(matches: &ArgMatches) -> UResult { if group.is_empty() { None } else { - match entries::grp2gid(group) { + match parse_gid_from_str(group) { Ok(g) => Some(g), - _ => { - return Err(USimpleError::new( - 1, - format!("invalid group: {}", group.quote()), - )) - } + Err(e) => return Err(USimpleError::new(1, e)), } } }; + Ok((dest_gid, raw_group)) +} + +fn parse_gid_and_uid(matches: &ArgMatches) -> UResult { + let (dest_gid, raw_group) = get_dest_gid(matches)?; + + // Handle --from option + let filter = if let Some(from_group) = matches.get_one::(options::FROM) { + match parse_gid_from_str(from_group) { + Ok(g) => IfFrom::Group(g), + Err(_) => { + return Err(USimpleError::new( + 1, + format!("invalid user: '{from_group}'"), + )); + } + } + } else { + IfFrom::All + }; + Ok(GidUidOwnerFilter { dest_gid, dest_uid: None, raw_owner: raw_group, - filter: IfFrom::All, + filter, }) } @@ -64,7 +98,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -73,7 +107,7 @@ pub fn uu_app() -> Command { Arg::new(options::HELP) .long(options::HELP) .help("Print help information.") - .action(ArgAction::Help) + .action(ArgAction::Help), ) .arg( Arg::new(options::verbosity::CHANGES) @@ -101,20 +135,6 @@ pub fn uu_app() -> Command { .help("output a diagnostic for every file processed") .action(ArgAction::SetTrue), ) - .arg( - Arg::new(options::dereference::DEREFERENCE) - .long(options::dereference::DEREFERENCE) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::dereference::NO_DEREFERENCE) - .short('h') - .long(options::dereference::NO_DEREFERENCE) - .help( - "affect symbolic links instead of any referenced file (useful only on systems that can change the ownership of a symlink)", - ) - .action(ArgAction::SetTrue), - ) .arg( Arg::new(options::preserve_root::PRESERVE) .long(options::preserve_root::PRESERVE) @@ -134,6 +154,12 @@ pub fn uu_app() -> Command { .value_hint(clap::ValueHint::FilePath) .help("use RFILE's group rather than specifying GROUP values"), ) + .arg( + Arg::new(options::FROM) + .long(options::FROM) + .value_name("GROUP") + .help("change the group only if its current group matches GROUP"), + ) .arg( Arg::new(options::RECURSIVE) .short('R') @@ -141,23 +167,6 @@ pub fn uu_app() -> Command { .help("operate on files and directories recursively") .action(ArgAction::SetTrue), ) - .arg( - Arg::new(options::traverse::TRAVERSE) - .short(options::traverse::TRAVERSE.chars().next().unwrap()) - .help("if a command line argument is a symbolic link to a directory, traverse it") - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::traverse::NO_TRAVERSE) - .short(options::traverse::NO_TRAVERSE.chars().next().unwrap()) - .help("do not traverse any symbolic links (default)") - .overrides_with_all([options::traverse::TRAVERSE, options::traverse::EVERY]) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::traverse::EVERY) - .short(options::traverse::EVERY.chars().next().unwrap()) - .help("traverse every symbolic link to a directory encountered") - .action(ArgAction::SetTrue), - ) + // Add common arguments with chgrp, chown & chmod + .args(uucore::perms::common_args()) } diff --git a/src/uu/chmod/Cargo.toml b/src/uu/chmod/Cargo.toml index e779469deda..09f1c531a90 100644 --- a/src/uu/chmod/Cargo.toml +++ b/src/uu/chmod/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_chmod" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "chmod ~ (uutils) change mode of FILE" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chmod" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chmod.rs" @@ -17,7 +20,7 @@ path = "src/chmod.rs" [dependencies] clap = { workspace = true } libc = { workspace = true } -uucore = { workspace = true, features = ["fs", "mode"] } +uucore = { workspace = true, features = ["entries", "fs", "mode", "perms"] } [[bin]] name = "chmod" diff --git a/src/uu/chmod/chmod.md b/src/uu/chmod/chmod.md index d6c2ed2d8e3..10ddb48a2ed 100644 --- a/src/uu/chmod/chmod.md +++ b/src/uu/chmod/chmod.md @@ -13,4 +13,4 @@ With --reference, change the mode of each FILE to that of RFILE. ## After Help -Each MODE is of the form '[ugoa]*([-+=]([rwxXst]*|[ugo]))+|[-+=]?[0-7]+'. +Each MODE is of the form `[ugoa]*([-+=]([rwxXst]*|[ugo]))+|[-+=]?[0-7]+`. diff --git a/src/uu/chmod/src/chmod.rs b/src/uu/chmod/src/chmod.rs index 31663b1af9c..dfe30485919 100644 --- a/src/uu/chmod/src/chmod.rs +++ b/src/uu/chmod/src/chmod.rs @@ -5,17 +5,18 @@ // spell-checker:ignore (ToDO) Chmoder cmode fmode fperm fref ugoa RFILE RFILE's -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use std::ffi::OsString; use std::fs; use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::path::Path; use uucore::display::Quotable; -use uucore::error::{set_exit_code, ExitCode, UResult, USimpleError, UUsageError}; +use uucore::error::{ExitCode, UResult, USimpleError, UUsageError, set_exit_code}; use uucore::fs::display_permissions_unix; use uucore::libc::mode_t; #[cfg(not(windows))] use uucore::mode; +use uucore::perms::{TraverseSymlinks, configure_symlink_and_recursion}; use uucore::{format_usage, help_about, help_section, help_usage, show, show_error}; const ABOUT: &str = help_about!("chmod.md"); @@ -23,6 +24,7 @@ const USAGE: &str = help_usage!("chmod.md"); const LONG_USAGE: &str = help_section!("after help", "chmod.md"); mod options { + pub const HELP: &str = "help"; pub const CHANGES: &str = "changes"; pub const QUIET: &str = "quiet"; // visible_alias("silent") pub const VERBOSE: &str = "verbose"; @@ -98,15 +100,14 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let quiet = matches.get_flag(options::QUIET); let verbose = matches.get_flag(options::VERBOSE); let preserve_root = matches.get_flag(options::PRESERVE_ROOT); - let recursive = matches.get_flag(options::RECURSIVE); let fmode = match matches.get_one::(options::REFERENCE) { Some(fref) => match fs::metadata(fref) { - Ok(meta) => Some(meta.mode()), + Ok(meta) => Some(meta.mode() & 0o7777), Err(err) => { return Err(USimpleError::new( 1, - format!("cannot stat attributes of {}: {}", fref.quote(), err), - )) + format!("cannot stat attributes of {}: {err}", fref.quote()), + )); } }, None => None, @@ -137,6 +138,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { return Err(UUsageError::new(1, "missing operand".to_string())); } + let (recursive, dereference, traverse_symlinks) = + configure_symlink_and_recursion(&matches, TraverseSymlinks::First)?; + let chmoder = Chmoder { changes, quiet, @@ -145,6 +149,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { recursive, fmode, cmode, + traverse_symlinks, + dereference, }; chmoder.chmod(&files) @@ -152,12 +158,19 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .args_override_self(true) .infer_long_args(true) .no_binary_name(true) + .disable_help_flag(true) + .arg( + Arg::new(options::HELP) + .long(options::HELP) + .help("Print help information.") + .action(ArgAction::Help), + ) .arg( Arg::new(options::CHANGES) .long(options::CHANGES) @@ -206,9 +219,10 @@ pub fn uu_app() -> Command { .help("use RFILE's mode instead of MODE values"), ) .arg( - Arg::new(options::MODE).required_unless_present(options::REFERENCE), // It would be nice if clap could parse with delimiter, e.g. "g-x,u+x", - // however .multiple_occurrences(true) cannot be used here because FILE already needs that. - // Only one positional argument with .multiple_occurrences(true) set is allowed per command + Arg::new(options::MODE).required_unless_present(options::REFERENCE), + // It would be nice if clap could parse with delimiter, e.g. "g-x,u+x", + // however .multiple_occurrences(true) cannot be used here because FILE already needs that. + // Only one positional argument with .multiple_occurrences(true) set is allowed per command ) .arg( Arg::new(options::FILE) @@ -216,6 +230,8 @@ pub fn uu_app() -> Command { .action(ArgAction::Append) .value_hint(clap::ValueHint::AnyPath), ) + // Add common arguments with chgrp, chown & chmod + .args(uucore::perms::common_args()) } struct Chmoder { @@ -226,6 +242,8 @@ struct Chmoder { recursive: bool, fmode: Option, cmode: Option, + traverse_symlinks: TraverseSymlinks, + dereference: bool, } impl Chmoder { @@ -237,12 +255,23 @@ impl Chmoder { let file = Path::new(filename); if !file.exists() { if file.is_symlink() { + if !self.dereference && !self.recursive { + // The file is a symlink and we should not follow it + // Don't try to change the mode of the symlink itself + continue; + } + if self.recursive && self.traverse_symlinks == TraverseSymlinks::None { + continue; + } + if !self.quiet { show!(USimpleError::new( 1, format!("cannot operate on dangling symlink {}", filename.quote()), )); + set_exit_code(1); } + if self.verbose { println!( "failed to change mode of {} from 0000 (---------) to 1500 (r-x-----T)", @@ -262,14 +291,19 @@ impl Chmoder { // So we set the exit code, because it hasn't been set yet if `self.quiet` is true. set_exit_code(1); continue; + } else if !self.dereference && file.is_symlink() { + // The file is a symlink and we should not follow it + // chmod 755 --no-dereference a/link + // should not change the permissions in this case + continue; } if self.recursive && self.preserve_root && filename == "/" { return Err(USimpleError::new( 1, format!( - "it is dangerous to operate recursively on {}\nuse --no-preserve-root to override this failsafe", + "it is dangerous to operate recursively on {}\nchmod: use --no-preserve-root to override this failsafe", filename.quote() - ) + ), )); } if self.recursive { @@ -283,11 +317,23 @@ impl Chmoder { fn walk_dir(&self, file_path: &Path) -> UResult<()> { let mut r = self.chmod_file(file_path); - if !file_path.is_symlink() && file_path.is_dir() { + // Determine whether to traverse symlinks based on `self.traverse_symlinks` + let should_follow_symlink = match self.traverse_symlinks { + TraverseSymlinks::All => true, + TraverseSymlinks::First => { + file_path == file_path.canonicalize().unwrap_or(file_path.to_path_buf()) + } + TraverseSymlinks::None => false, + }; + + // If the path is a directory (or we should follow symlinks), recurse into it + if (!file_path.is_symlink() || should_follow_symlink) && file_path.is_dir() { for dir_entry in file_path.read_dir()? { let path = dir_entry?.path(); if !path.is_symlink() { r = self.walk_dir(path.as_path()); + } else if should_follow_symlink { + r = self.chmod_file(path.as_path()).and(r); } } } @@ -303,31 +349,36 @@ impl Chmoder { } #[cfg(unix)] fn chmod_file(&self, file: &Path) -> UResult<()> { - use uucore::mode::get_umask; + use uucore::{mode::get_umask, perms::get_metadata}; - let fperm = match fs::metadata(file) { + let metadata = get_metadata(file, self.dereference); + + let fperm = match metadata { Ok(meta) => meta.mode() & 0o7777, Err(err) => { - if file.is_symlink() { + // Handle dangling symlinks or other errors + return if file.is_symlink() && !self.dereference { if self.verbose { println!( "neither symbolic link {} nor referent has been changed", file.quote() ); } - return Ok(()); + Ok(()) // Skip dangling symlinks } else if err.kind() == std::io::ErrorKind::PermissionDenied { // These two filenames would normally be conditionally // quoted, but GNU's tests expect them to always be quoted - return Err(USimpleError::new( + Err(USimpleError::new( 1, format!("{}: Permission denied", file.quote()), - )); + )) } else { - return Err(USimpleError::new(1, format!("{}: {}", file.quote(), err))); - } + Err(USimpleError::new(1, format!("{}: {err}", file.quote()))) + }; } }; + + // Determine the new permissions to apply match self.fmode { Some(mode) => self.change_file(fperm, mode, file)?, None => { @@ -390,24 +441,21 @@ impl Chmoder { if fperm == mode { if self.verbose && !self.changes { println!( - "mode of {} retained as {:04o} ({})", + "mode of {} retained as {fperm:04o} ({})", file.quote(), - fperm, display_permissions_unix(fperm as mode_t, false), ); } Ok(()) } else if let Err(err) = fs::set_permissions(file, fs::Permissions::from_mode(mode)) { if !self.quiet { - show_error!("{}", err); + show_error!("{err}"); } if self.verbose { println!( - "failed to change mode of file {} from {:04o} ({}) to {:04o} ({})", + "failed to change mode of file {} from {fperm:04o} ({}) to {mode:04o} ({})", file.quote(), - fperm, display_permissions_unix(fperm as mode_t, false), - mode, display_permissions_unix(mode as mode_t, false) ); } @@ -415,11 +463,9 @@ impl Chmoder { } else { if self.verbose || self.changes { println!( - "mode of {} changed from {:04o} ({}) to {:04o} ({})", + "mode of {} changed from {fperm:04o} ({}) to {mode:04o} ({})", file.quote(), - fperm, display_permissions_unix(fperm as mode_t, false), - mode, display_permissions_unix(mode as mode_t, false) ); } diff --git a/src/uu/chown/Cargo.toml b/src/uu/chown/Cargo.toml index dfa9dba32a1..dcf7c445412 100644 --- a/src/uu/chown/Cargo.toml +++ b/src/uu/chown/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_chown" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "chown ~ (uutils) change the ownership of FILE" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chown" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chown.rs" diff --git a/src/uu/chown/src/chown.rs b/src/uu/chown/src/chown.rs index 0e9b8b2423c..4389d92f663 100644 --- a/src/uu/chown/src/chown.rs +++ b/src/uu/chown/src/chown.rs @@ -7,12 +7,12 @@ use uucore::display::Quotable; pub use uucore::entries::{self, Group, Locate, Passwd}; -use uucore::perms::{chown_base, options, GidUidOwnerFilter, IfFrom}; +use uucore::perms::{GidUidOwnerFilter, IfFrom, chown_base, options}; use uucore::{format_usage, help_about, help_usage}; use uucore::error::{FromIo, UResult, USimpleError}; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; use std::fs; use std::os::unix::fs::MetadataExt; @@ -78,7 +78,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -96,25 +96,6 @@ pub fn uu_app() -> Command { .help("like verbose but report only when a change is made") .action(ArgAction::SetTrue), ) - .arg( - Arg::new(options::dereference::DEREFERENCE) - .long(options::dereference::DEREFERENCE) - .help( - "affect the referent of each symbolic link (this is the default), \ - rather than the symbolic link itself", - ) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::dereference::NO_DEREFERENCE) - .short('h') - .long(options::dereference::NO_DEREFERENCE) - .help( - "affect symbolic links instead of any referenced file \ - (useful only on systems that can change the ownership of a symlink)", - ) - .action(ArgAction::SetTrue), - ) .arg( Arg::new(options::FROM) .long(options::FROM) @@ -165,27 +146,6 @@ pub fn uu_app() -> Command { .long(options::verbosity::SILENT) .action(ArgAction::SetTrue), ) - .arg( - Arg::new(options::traverse::TRAVERSE) - .short(options::traverse::TRAVERSE.chars().next().unwrap()) - .help("if a command line argument is a symbolic link to a directory, traverse it") - .overrides_with_all([options::traverse::EVERY, options::traverse::NO_TRAVERSE]) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::traverse::EVERY) - .short(options::traverse::EVERY.chars().next().unwrap()) - .help("traverse every symbolic link to a directory encountered") - .overrides_with_all([options::traverse::TRAVERSE, options::traverse::NO_TRAVERSE]) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::traverse::NO_TRAVERSE) - .short(options::traverse::NO_TRAVERSE.chars().next().unwrap()) - .help("do not traverse any symbolic links (default)") - .overrides_with_all([options::traverse::TRAVERSE, options::traverse::EVERY]) - .action(ArgAction::SetTrue), - ) .arg( Arg::new(options::verbosity::VERBOSE) .long(options::verbosity::VERBOSE) @@ -193,6 +153,8 @@ pub fn uu_app() -> Command { .help("output a diagnostic for every file processed") .action(ArgAction::SetTrue), ) + // Add common arguments with chgrp, chown & chmod + .args(uucore::perms::common_args()) } /// Parses the user string to extract the UID. diff --git a/src/uu/chroot/Cargo.toml b/src/uu/chroot/Cargo.toml index 12533b6542a..4d302d95f06 100644 --- a/src/uu/chroot/Cargo.toml +++ b/src/uu/chroot/Cargo.toml @@ -1,21 +1,25 @@ [package] name = "uu_chroot" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "chroot ~ (uutils) run COMMAND under a new root directory" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chroot" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chroot.rs" [dependencies] clap = { workspace = true } +thiserror = { workspace = true } uucore = { workspace = true, features = ["entries", "fs"] } [[bin]] diff --git a/src/uu/chroot/src/chroot.rs b/src/uu/chroot/src/chroot.rs index fb20b0ccc46..3f7c0886b5f 100644 --- a/src/uu/chroot/src/chroot.rs +++ b/src/uu/chroot/src/chroot.rs @@ -7,30 +7,153 @@ mod error; use crate::error::ChrootError; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use std::ffi::CString; use std::io::Error; use std::os::unix::prelude::OsStrExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process; -use uucore::error::{set_exit_code, UClapError, UResult, UUsageError}; -use uucore::fs::{canonicalize, MissingHandling, ResolveMode}; +use uucore::entries::{Locate, Passwd, grp2gid, usr2uid}; +use uucore::error::{UClapError, UResult, UUsageError, set_exit_code}; +use uucore::fs::{MissingHandling, ResolveMode, canonicalize}; use uucore::libc::{self, chroot, setgid, setgroups, setuid}; -use uucore::{entries, format_usage, help_about, help_usage}; +use uucore::{format_usage, help_about, help_usage, show}; static ABOUT: &str = help_about!("chroot.md"); static USAGE: &str = help_usage!("chroot.md"); mod options { pub const NEWROOT: &str = "newroot"; - pub const USER: &str = "user"; - pub const GROUP: &str = "group"; pub const GROUPS: &str = "groups"; pub const USERSPEC: &str = "userspec"; pub const COMMAND: &str = "command"; pub const SKIP_CHDIR: &str = "skip-chdir"; } +/// A user and group specification, where each is optional. +enum UserSpec { + NeitherGroupNorUser, + UserOnly(String), + GroupOnly(String), + UserAndGroup(String, String), +} + +struct Options { + /// Path to the new root directory. + newroot: PathBuf, + /// Whether to change to the new root directory. + skip_chdir: bool, + /// List of groups under which the command will be run. + groups: Option>, + /// The user and group (each optional) under which the command will be run. + userspec: Option, +} + +/// Parse a user and group from the argument to `--userspec`. +/// +/// The `spec` must be of the form `[USER][:[GROUP]]`, otherwise an +/// error is returned. +fn parse_userspec(spec: &str) -> UserSpec { + match spec.split_once(':') { + // "" + None if spec.is_empty() => UserSpec::NeitherGroupNorUser, + // "usr" + None => UserSpec::UserOnly(spec.to_string()), + // ":" + Some(("", "")) => UserSpec::NeitherGroupNorUser, + // ":grp" + Some(("", grp)) => UserSpec::GroupOnly(grp.to_string()), + // "usr:" + Some((usr, "")) => UserSpec::UserOnly(usr.to_string()), + // "usr:grp" + Some((usr, grp)) => UserSpec::UserAndGroup(usr.to_string(), grp.to_string()), + } +} + +// Pre-condition: `list_str` is non-empty. +fn parse_group_list(list_str: &str) -> Result, ChrootError> { + let split: Vec<&str> = list_str.split(',').collect(); + if split.len() == 1 { + let name = split[0].trim(); + if name.is_empty() { + // --groups=" " + // chroot: invalid group ‘ ’ + Err(ChrootError::InvalidGroup(name.to_string())) + } else { + // --groups="blah" + Ok(vec![name.to_string()]) + } + } else if split.iter().all(|s| s.is_empty()) { + // --groups="," + // chroot: invalid group list ‘,’ + Err(ChrootError::InvalidGroupList(list_str.to_string())) + } else { + let mut result = vec![]; + let mut err = false; + for name in split { + let trimmed_name = name.trim(); + if trimmed_name.is_empty() { + if name.is_empty() { + // --groups="," + continue; + } else { + // --groups=", " + // chroot: invalid group ‘ ’ + show!(ChrootError::InvalidGroup(name.to_string())); + err = true; + } + } else { + // TODO Figure out a better condition here. + if trimmed_name.starts_with(char::is_numeric) + && trimmed_name.ends_with(|c: char| !c.is_numeric()) + { + // --groups="0trail" + // chroot: invalid group ‘0trail’ + show!(ChrootError::InvalidGroup(name.to_string())); + err = true; + } else { + result.push(trimmed_name.to_string()); + } + } + } + if err { + Err(ChrootError::GroupsParsingFailed) + } else { + Ok(result) + } + } +} + +impl Options { + /// Parse parameters from the command-line arguments. + fn from(matches: &clap::ArgMatches) -> UResult { + let newroot = match matches.get_one::(options::NEWROOT) { + Some(v) => Path::new(v).to_path_buf(), + None => return Err(ChrootError::MissingNewRoot.into()), + }; + let groups = match matches.get_one::(options::GROUPS) { + None => None, + Some(s) => { + if s.is_empty() { + Some(vec![]) + } else { + Some(parse_group_list(s)?) + } + } + }; + let skip_chdir = matches.get_flag(options::SKIP_CHDIR); + let userspec = matches + .get_one::(options::USERSPEC) + .map(|s| parse_userspec(s)); + Ok(Self { + newroot, + skip_chdir, + groups, + userspec, + }) + } +} + #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let matches = uu_app().try_get_matches_from(args).with_exit_code(125)?; @@ -39,17 +162,17 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let default_option: &'static str = "-i"; let user_shell = std::env::var("SHELL"); - let newroot: &Path = match matches.get_one::(options::NEWROOT) { - Some(v) => Path::new(v), - None => return Err(ChrootError::MissingNewRoot.into()), - }; + let options = Options::from(&matches)?; - let skip_chdir = matches.get_flag(options::SKIP_CHDIR); // We are resolving the path in case it is a symlink or /. or /../ - if skip_chdir - && canonicalize(newroot, MissingHandling::Normal, ResolveMode::Logical) - .unwrap() - .to_str() + if options.skip_chdir + && canonicalize( + &options.newroot, + MissingHandling::Normal, + ResolveMode::Logical, + ) + .unwrap() + .to_str() != Some("/") { return Err(UUsageError::new( @@ -58,8 +181,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { )); } - if !newroot.is_dir() { - return Err(ChrootError::NoSuchDirectory(format!("{}", newroot.display())).into()); + if !options.newroot.is_dir() { + return Err(ChrootError::NoSuchDirectory(format!("{}", options.newroot.display())).into()); } let commands = match matches.get_many::(options::COMMAND) { @@ -85,7 +208,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let chroot_args = &command[1..]; // NOTE: Tests can only trigger code beyond this point if they're invoked with root permissions - set_context(newroot, &matches)?; + set_context(&options)?; let pstatus = match process::Command::new(chroot_command) .args(chroot_args) @@ -98,7 +221,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } else { ChrootError::CommandFailed(command[0].to_string(), e) } - .into()) + .into()); } }; @@ -113,7 +236,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -125,35 +248,17 @@ pub fn uu_app() -> Command { .required(true) .index(1), ) - .arg( - Arg::new(options::USER) - .short('u') - .long(options::USER) - .help("User (ID or name) to switch before running the program") - .value_name("USER"), - ) - .arg( - Arg::new(options::GROUP) - .short('g') - .long(options::GROUP) - .help("Group (ID or name) to switch to") - .value_name("GROUP"), - ) .arg( Arg::new(options::GROUPS) - .short('G') .long(options::GROUPS) + .overrides_with(options::GROUPS) .help("Comma-separated list of groups to switch to") .value_name("GROUP1,GROUP2..."), ) .arg( Arg::new(options::USERSPEC) .long(options::USERSPEC) - .help( - "Colon-separated user and group to switch to. \ - Same as -u USER -g GROUP. \ - Userspec has higher preference than -u and/or -g", - ) + .help("Colon-separated user and group to switch to.") .value_name("USER:GROUP"), ) .arg( @@ -175,119 +280,178 @@ pub fn uu_app() -> Command { ) } -fn set_context(root: &Path, options: &clap::ArgMatches) -> UResult<()> { - let userspec_str = options.get_one::(options::USERSPEC); - let user_str = options - .get_one::(options::USER) - .map(|s| s.as_str()) - .unwrap_or_default(); - let group_str = options - .get_one::(options::GROUP) - .map(|s| s.as_str()) - .unwrap_or_default(); - let groups_str = options - .get_one::(options::GROUPS) - .map(|s| s.as_str()) - .unwrap_or_default(); - let skip_chdir = options.contains_id(options::SKIP_CHDIR); - let userspec = match userspec_str { - Some(u) => { - let s: Vec<&str> = u.split(':').collect(); - if s.len() != 2 || s.iter().any(|&spec| spec.is_empty()) { - return Err(ChrootError::InvalidUserspec(u.to_string()).into()); - }; - s - } - None => Vec::new(), - }; - - let (user, group) = if userspec.is_empty() { - (user_str, group_str) - } else { - (userspec[0], userspec[1]) - }; +/// Get the UID for the given username, falling back to numeric parsing. +/// +/// According to the documentation of GNU `chroot`, "POSIX requires that +/// these commands first attempt to resolve the specified string as a +/// name, and only once that fails, then try to interpret it as an ID." +fn name_to_uid(name: &str) -> Result { + match usr2uid(name) { + Ok(uid) => Ok(uid), + Err(_) => name + .parse::() + .map_err(|_| ChrootError::NoSuchUser), + } +} - enter_chroot(root, skip_chdir)?; +/// Get the GID for the given group name, falling back to numeric parsing. +/// +/// According to the documentation of GNU `chroot`, "POSIX requires that +/// these commands first attempt to resolve the specified string as a +/// name, and only once that fails, then try to interpret it as an ID." +fn name_to_gid(name: &str) -> Result { + match grp2gid(name) { + Ok(gid) => Ok(gid), + Err(_) => name + .parse::() + .map_err(|_| ChrootError::NoSuchGroup), + } +} - set_groups_from_str(groups_str)?; - set_main_group(group)?; - set_user(user)?; - Ok(()) +/// Get the list of group IDs for the given user. +/// +/// According to the GNU documentation, "the supplementary groups are +/// set according to the system defined list for that user". This +/// function gets that list. +fn supplemental_gids(uid: libc::uid_t) -> Vec { + match Passwd::locate(uid) { + Err(_) => vec![], + Ok(passwd) => passwd.belongs_to(), + } } -fn enter_chroot(root: &Path, skip_chdir: bool) -> UResult<()> { - let err = unsafe { - chroot( - CString::new(root.as_os_str().as_bytes().to_vec()) - .unwrap() - .as_bytes_with_nul() - .as_ptr() as *const libc::c_char, - ) - }; +/// Set the supplemental group IDs for this process. +fn set_supplemental_gids(gids: &[libc::gid_t]) -> std::io::Result<()> { + #[cfg(any(target_vendor = "apple", target_os = "freebsd", target_os = "openbsd"))] + let n = gids.len() as libc::c_int; + #[cfg(any(target_os = "linux", target_os = "android"))] + let n = gids.len() as libc::size_t; + let err = unsafe { setgroups(n, gids.as_ptr()) }; + if err == 0 { + Ok(()) + } else { + Err(Error::last_os_error()) + } +} +/// Set the group ID of this process. +fn set_gid(gid: libc::gid_t) -> std::io::Result<()> { + let err = unsafe { setgid(gid) }; if err == 0 { - if !skip_chdir { - std::env::set_current_dir(root).unwrap(); - } Ok(()) } else { - Err(ChrootError::CannotEnter(format!("{}", root.display()), Error::last_os_error()).into()) + Err(Error::last_os_error()) } } -fn set_main_group(group: &str) -> UResult<()> { - if !group.is_empty() { - let group_id = match entries::grp2gid(group) { - Ok(g) => g, - _ => return Err(ChrootError::NoSuchGroup(group.to_string()).into()), - }; - let err = unsafe { setgid(group_id) }; - if err != 0 { - return Err( - ChrootError::SetGidFailed(group_id.to_string(), Error::last_os_error()).into(), - ); - } +/// Set the user ID of this process. +fn set_uid(uid: libc::uid_t) -> std::io::Result<()> { + let err = unsafe { setuid(uid) }; + if err == 0 { + Ok(()) + } else { + Err(Error::last_os_error()) } - Ok(()) } -#[cfg(any(target_vendor = "apple", target_os = "freebsd", target_os = "openbsd"))] -fn set_groups(groups: &[libc::gid_t]) -> libc::c_int { - unsafe { setgroups(groups.len() as libc::c_int, groups.as_ptr()) } +/// What to do when the `--groups` argument is missing. +enum Strategy { + /// Do nothing. + Nothing, + /// Use the list of supplemental groups for the given user. + /// + /// If the `bool` parameter is `false` and the list of groups for + /// the given user is empty, then this will result in an error. + FromUID(libc::uid_t, bool), +} + +/// Set supplemental groups when the `--groups` argument is not specified. +fn handle_missing_groups(strategy: Strategy) -> Result<(), ChrootError> { + match strategy { + Strategy::Nothing => Ok(()), + Strategy::FromUID(uid, false) => { + let gids = supplemental_gids(uid); + if gids.is_empty() { + Err(ChrootError::NoGroupSpecified(uid)) + } else { + set_supplemental_gids(&gids).map_err(ChrootError::SetGroupsFailed) + } + } + Strategy::FromUID(uid, true) => { + let gids = supplemental_gids(uid); + set_supplemental_gids(&gids).map_err(ChrootError::SetGroupsFailed) + } + } } -#[cfg(any(target_os = "linux", target_os = "android"))] -fn set_groups(groups: &[libc::gid_t]) -> libc::c_int { - unsafe { setgroups(groups.len() as libc::size_t, groups.as_ptr()) } +/// Set supplemental groups for this process. +fn set_supplemental_gids_with_strategy( + strategy: Strategy, + groups: Option<&Vec>, +) -> Result<(), ChrootError> { + match groups { + None => handle_missing_groups(strategy), + Some(groups) => { + let mut gids = vec![]; + for group in groups { + gids.push(name_to_gid(group)?); + } + set_supplemental_gids(&gids).map_err(ChrootError::SetGroupsFailed) + } + } } -fn set_groups_from_str(groups: &str) -> UResult<()> { - if !groups.is_empty() { - let mut groups_vec = vec![]; - for group in groups.split(',') { - let gid = match entries::grp2gid(group) { - Ok(g) => g, - Err(_) => return Err(ChrootError::NoSuchGroup(group.to_string()).into()), - }; - groups_vec.push(gid); +/// Change the root, set the user ID, and set the group IDs for this process. +fn set_context(options: &Options) -> UResult<()> { + enter_chroot(&options.newroot, options.skip_chdir)?; + match &options.userspec { + None | Some(UserSpec::NeitherGroupNorUser) => { + let strategy = Strategy::Nothing; + set_supplemental_gids_with_strategy(strategy, options.groups.as_ref())?; } - let err = set_groups(&groups_vec); - if err != 0 { - return Err(ChrootError::SetGroupsFailed(Error::last_os_error()).into()); + Some(UserSpec::UserOnly(user)) => { + let uid = name_to_uid(user)?; + let gid = uid as libc::gid_t; + let strategy = Strategy::FromUID(uid, false); + set_supplemental_gids_with_strategy(strategy, options.groups.as_ref())?; + set_gid(gid).map_err(|e| ChrootError::SetGidFailed(user.to_string(), e))?; + set_uid(uid).map_err(|e| ChrootError::SetUserFailed(user.to_string(), e))?; + } + Some(UserSpec::GroupOnly(group)) => { + let gid = name_to_gid(group)?; + let strategy = Strategy::Nothing; + set_supplemental_gids_with_strategy(strategy, options.groups.as_ref())?; + set_gid(gid).map_err(|e| ChrootError::SetGidFailed(group.to_string(), e))?; + } + Some(UserSpec::UserAndGroup(user, group)) => { + let uid = name_to_uid(user)?; + let gid = name_to_gid(group)?; + let strategy = Strategy::FromUID(uid, true); + set_supplemental_gids_with_strategy(strategy, options.groups.as_ref())?; + set_gid(gid).map_err(|e| ChrootError::SetGidFailed(group.to_string(), e))?; + set_uid(uid).map_err(|e| ChrootError::SetUserFailed(user.to_string(), e))?; } } Ok(()) } -fn set_user(user: &str) -> UResult<()> { - if !user.is_empty() { - let user_id = entries::usr2uid(user).unwrap(); - let err = unsafe { setuid(user_id as libc::uid_t) }; - if err != 0 { - return Err( - ChrootError::SetUserFailed(user.to_string(), Error::last_os_error()).into(), - ); +fn enter_chroot(root: &Path, skip_chdir: bool) -> UResult<()> { + let err = unsafe { + chroot( + CString::new(root.as_os_str().as_bytes().to_vec()) + .unwrap() + .as_bytes_with_nul() + .as_ptr() + .cast::(), + ) + }; + + if err == 0 { + if !skip_chdir { + std::env::set_current_dir("/").unwrap(); } + Ok(()) + } else { + Err(ChrootError::CannotEnter(format!("{}", root.display()), Error::last_os_error()).into()) } - Ok(()) } diff --git a/src/uu/chroot/src/error.rs b/src/uu/chroot/src/error.rs index 526f1a75a43..78fd7ad64e7 100644 --- a/src/uu/chroot/src/error.rs +++ b/src/uu/chroot/src/error.rs @@ -4,47 +4,71 @@ // file that was distributed with this source code. // spell-checker:ignore NEWROOT Userspec userspec //! Errors returned by chroot. -use std::fmt::Display; use std::io::Error; +use thiserror::Error; use uucore::display::Quotable; use uucore::error::UError; +use uucore::libc; /// Errors that can happen while executing chroot. -#[derive(Debug)] +#[derive(Debug, Error)] pub enum ChrootError { /// Failed to enter the specified directory. - CannotEnter(String, Error), + #[error("cannot chroot to {dir}: {err}", dir = .0.quote(), err = .1)] + CannotEnter(String, #[source] Error), /// Failed to execute the specified command. - CommandFailed(String, Error), + #[error("failed to run command {cmd}: {err}", cmd = .0.to_string().quote(), err = .1)] + CommandFailed(String, #[source] Error), /// Failed to find the specified command. - CommandNotFound(String, Error), + #[error("failed to run command {cmd}: {err}", cmd = .0.to_string().quote(), err = .1)] + CommandNotFound(String, #[source] Error), - /// The given user and group specification was invalid. - InvalidUserspec(String), + #[error("--groups parsing failed")] + GroupsParsingFailed, + + #[error("invalid group: {group}", group = .0.quote())] + InvalidGroup(String), + + #[error("invalid group list: {list}", list = .0.quote())] + InvalidGroupList(String), /// The new root directory was not given. + #[error( + "Missing operand: NEWROOT\nTry '{0} --help' for more information.", + uucore::execution_phrase() + )] MissingNewRoot, + #[error("no group specified for unknown uid: {0}")] + NoGroupSpecified(libc::uid_t), + + /// Failed to find the specified user. + #[error("invalid user")] + NoSuchUser, + /// Failed to find the specified group. - NoSuchGroup(String), + #[error("invalid group")] + NoSuchGroup, /// The given directory does not exist. + #[error("cannot change root directory to {dir}: no such directory", dir = .0.quote())] NoSuchDirectory(String), /// The call to `setgid()` failed. - SetGidFailed(String, Error), + #[error("cannot set gid to {gid}: {err}", gid = .0, err = .1)] + SetGidFailed(String, #[source] Error), /// The call to `setgroups()` failed. + #[error("cannot set groups: {0}")] SetGroupsFailed(Error), /// The call to `setuid()` failed. - SetUserFailed(String, Error), + #[error("cannot set user to {user}: {err}", user = .0.maybe_quote(), err = .1)] + SetUserFailed(String, #[source] Error), } -impl std::error::Error for ChrootError {} - impl UError for ChrootError { // 125 if chroot itself fails // 126 if command is found but cannot be invoked @@ -57,31 +81,3 @@ impl UError for ChrootError { } } } - -impl Display for ChrootError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::CannotEnter(s, e) => write!(f, "cannot chroot to {}: {}", s.quote(), e,), - Self::CommandFailed(s, e) | Self::CommandNotFound(s, e) => { - write!(f, "failed to run command {}: {}", s.to_string().quote(), e,) - } - Self::InvalidUserspec(s) => write!(f, "invalid userspec: {}", s.quote(),), - Self::MissingNewRoot => write!( - f, - "Missing operand: NEWROOT\nTry '{} --help' for more information.", - uucore::execution_phrase(), - ), - Self::NoSuchGroup(s) => write!(f, "no such group: {}", s.maybe_quote(),), - Self::NoSuchDirectory(s) => write!( - f, - "cannot change root directory to {}: no such directory", - s.quote(), - ), - Self::SetGidFailed(s, e) => write!(f, "cannot set gid to {s}: {e}"), - Self::SetGroupsFailed(e) => write!(f, "cannot set groups: {e}"), - Self::SetUserFailed(s, e) => { - write!(f, "cannot set user to {}: {}", s.maybe_quote(), e) - } - } - } -} diff --git a/src/uu/cksum/Cargo.toml b/src/uu/cksum/Cargo.toml index 47b4d73592a..c49288aa9d4 100644 --- a/src/uu/cksum/Cargo.toml +++ b/src/uu/cksum/Cargo.toml @@ -1,23 +1,27 @@ [package] name = "uu_cksum" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "cksum ~ (uutils) display CRC and size of input" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/cksum" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/cksum.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true, features = ["sum"] } +uucore = { workspace = true, features = ["checksum", "encoding", "sum"] } hex = { workspace = true } +regex = { workspace = true } [[bin]] name = "cksum" diff --git a/src/uu/cksum/cksum.md b/src/uu/cksum/cksum.md index 4b0d25f32c3..5ca83b40150 100644 --- a/src/uu/cksum/cksum.md +++ b/src/uu/cksum/cksum.md @@ -13,6 +13,7 @@ DIGEST determines the digest algorithm and default output format: - `sysv`: (equivalent to sum -s) - `bsd`: (equivalent to sum -r) - `crc`: (equivalent to cksum) +- `crc32b`: (only available through cksum) - `md5`: (equivalent to md5sum) - `sha1`: (equivalent to sha1sum) - `sha224`: (equivalent to sha224sum) diff --git a/src/uu/cksum/src/cksum.rs b/src/uu/cksum/src/cksum.rs index 36dfbbe1e3d..a1a9115d9a0 100644 --- a/src/uu/cksum/src/cksum.rs +++ b/src/uu/cksum/src/cksum.rs @@ -4,141 +4,48 @@ // file that was distributed with this source code. // spell-checker:ignore (ToDO) fname, algo -use clap::{crate_version, value_parser, Arg, ArgAction, Command}; -use hex::decode; -use hex::encode; -use std::error::Error; -use std::ffi::OsStr; -use std::fmt::Display; +use clap::builder::ValueParser; +use clap::{Arg, ArgAction, Command, value_parser}; +use std::ffi::{OsStr, OsString}; use std::fs::File; -use std::io::{self, stdin, stdout, BufReader, Read, Write}; +use std::io::{self, BufReader, Read, Write, stdin, stdout}; use std::iter; use std::path::Path; +use uucore::checksum::{ + ALGORITHM_OPTIONS_BLAKE2B, ALGORITHM_OPTIONS_BSD, ALGORITHM_OPTIONS_CRC, + ALGORITHM_OPTIONS_CRC32B, ALGORITHM_OPTIONS_SYSV, ChecksumError, ChecksumOptions, + ChecksumVerbose, SUPPORTED_ALGORITHMS, calculate_blake2b_length, detect_algo, digest_reader, + perform_checksum_validation, +}; use uucore::{ - error::{FromIo, UError, UResult, USimpleError}, - format_usage, help_about, help_section, help_usage, show, - sum::{ - div_ceil, Blake2b, Digest, DigestWriter, Md5, Sha1, Sha224, Sha256, Sha384, Sha512, Sm3, - BSD, CRC, SYSV, - }, + encoding, + error::{FromIo, UResult, USimpleError}, + format_usage, help_about, help_section, help_usage, + line_ending::LineEnding, + os_str_as_bytes, show, + sum::Digest, }; const USAGE: &str = help_usage!("cksum.md"); const ABOUT: &str = help_about!("cksum.md"); const AFTER_HELP: &str = help_section!("after help", "cksum.md"); -const ALGORITHM_OPTIONS_SYSV: &str = "sysv"; -const ALGORITHM_OPTIONS_BSD: &str = "bsd"; -const ALGORITHM_OPTIONS_CRC: &str = "crc"; -const ALGORITHM_OPTIONS_MD5: &str = "md5"; -const ALGORITHM_OPTIONS_SHA1: &str = "sha1"; -const ALGORITHM_OPTIONS_SHA224: &str = "sha224"; -const ALGORITHM_OPTIONS_SHA256: &str = "sha256"; -const ALGORITHM_OPTIONS_SHA384: &str = "sha384"; -const ALGORITHM_OPTIONS_SHA512: &str = "sha512"; -const ALGORITHM_OPTIONS_BLAKE2B: &str = "blake2b"; -const ALGORITHM_OPTIONS_SM3: &str = "sm3"; - -#[derive(Debug)] -enum CkSumError { - RawMultipleFiles, -} - -impl UError for CkSumError { - fn code(&self) -> i32 { - match self { - Self::RawMultipleFiles => 1, - } - } -} - -impl Error for CkSumError {} - -impl Display for CkSumError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::RawMultipleFiles => { - write!(f, "the --raw option is not supported with multiple files") - } - } - } -} - -fn detect_algo( - program: &str, - length: Option, -) -> (&'static str, Box, usize) { - match program { - ALGORITHM_OPTIONS_SYSV => ( - ALGORITHM_OPTIONS_SYSV, - Box::new(SYSV::new()) as Box, - 512, - ), - ALGORITHM_OPTIONS_BSD => ( - ALGORITHM_OPTIONS_BSD, - Box::new(BSD::new()) as Box, - 1024, - ), - ALGORITHM_OPTIONS_CRC => ( - ALGORITHM_OPTIONS_CRC, - Box::new(CRC::new()) as Box, - 256, - ), - ALGORITHM_OPTIONS_MD5 => ( - ALGORITHM_OPTIONS_MD5, - Box::new(Md5::new()) as Box, - 128, - ), - ALGORITHM_OPTIONS_SHA1 => ( - ALGORITHM_OPTIONS_SHA1, - Box::new(Sha1::new()) as Box, - 160, - ), - ALGORITHM_OPTIONS_SHA224 => ( - ALGORITHM_OPTIONS_SHA224, - Box::new(Sha224::new()) as Box, - 224, - ), - ALGORITHM_OPTIONS_SHA256 => ( - ALGORITHM_OPTIONS_SHA256, - Box::new(Sha256::new()) as Box, - 256, - ), - ALGORITHM_OPTIONS_SHA384 => ( - ALGORITHM_OPTIONS_SHA384, - Box::new(Sha384::new()) as Box, - 384, - ), - ALGORITHM_OPTIONS_SHA512 => ( - ALGORITHM_OPTIONS_SHA512, - Box::new(Sha512::new()) as Box, - 512, - ), - ALGORITHM_OPTIONS_BLAKE2B => ( - ALGORITHM_OPTIONS_BLAKE2B, - Box::new(if let Some(length) = length { - Blake2b::with_output_bytes(length) - } else { - Blake2b::new() - }) as Box, - 512, - ), - ALGORITHM_OPTIONS_SM3 => ( - ALGORITHM_OPTIONS_SM3, - Box::new(Sm3::new()) as Box, - 512, - ), - _ => unreachable!("unknown algorithm: clap should have prevented this case"), - } +#[derive(Debug, PartialEq)] +enum OutputFormat { + Hexadecimal, + Raw, + Base64, } struct Options { algo_name: &'static str, digest: Box, output_bits: usize, - untagged: bool, + tag: bool, // will cover the --untagged option length: Option, - raw: bool, + output_format: OutputFormat, + asterisk: bool, // if we display an asterisk or not (--binary/--text) + line_ending: LineEnding, } /// Calculate checksum @@ -153,8 +60,8 @@ where I: Iterator, { let files: Vec<_> = files.collect(); - if options.raw && files.len() > 1 { - return Err(Box::new(CkSumError::RawMultipleFiles)); + if options.output_format == OutputFormat::Raw && files.len() > 1 { + return Err(Box::new(ChecksumError::RawMultipleFiles)); } for filename in files { @@ -162,6 +69,8 @@ where let stdin_buf; let file_buf; let not_file = filename == OsStr::new("-"); + + // Handle the file input let mut file = BufReader::new(if not_file { stdin_buf = stdin(); Box::new(stdin_buf) as Box @@ -177,8 +86,7 @@ where }; Box::new(file_buf) as Box }); - let (sum, sz) = digest_read(&mut options.digest, &mut file, options.output_bits) - .map_err_context(|| "failed to read input".to_string())?; + if filename.is_dir() { show!(USimpleError::new( 1, @@ -186,172 +94,245 @@ where )); continue; } - if options.raw { - let bytes = match options.algo_name { - ALGORITHM_OPTIONS_CRC => sum.parse::().unwrap().to_be_bytes().to_vec(), - ALGORITHM_OPTIONS_SYSV | ALGORITHM_OPTIONS_BSD => { - sum.parse::().unwrap().to_be_bytes().to_vec() - } - _ => decode(sum).unwrap(), - }; - stdout().write_all(&bytes)?; - return Ok(()); - } + + let (sum_hex, sz) = + digest_reader(&mut options.digest, &mut file, false, options.output_bits) + .map_err_context(|| "failed to read input".to_string())?; + + let sum = match options.output_format { + OutputFormat::Raw => { + let bytes = match options.algo_name { + ALGORITHM_OPTIONS_CRC => sum_hex.parse::().unwrap().to_be_bytes().to_vec(), + ALGORITHM_OPTIONS_SYSV | ALGORITHM_OPTIONS_BSD => { + sum_hex.parse::().unwrap().to_be_bytes().to_vec() + } + _ => hex::decode(sum_hex).unwrap(), + }; + // Cannot handle multiple files anyway, output immediately. + stdout().write_all(&bytes)?; + return Ok(()); + } + OutputFormat::Hexadecimal => sum_hex, + OutputFormat::Base64 => match options.algo_name { + ALGORITHM_OPTIONS_CRC + | ALGORITHM_OPTIONS_CRC32B + | ALGORITHM_OPTIONS_SYSV + | ALGORITHM_OPTIONS_BSD => sum_hex, + _ => encoding::for_cksum::BASE64.encode(&hex::decode(sum_hex).unwrap()), + }, + }; // The BSD checksum output is 5 digit integer let bsd_width = 5; - match (options.algo_name, not_file) { - (ALGORITHM_OPTIONS_SYSV, true) => println!( - "{} {}", - sum.parse::().unwrap(), - div_ceil(sz, options.output_bits) + let (before_filename, should_print_filename, after_filename) = match options.algo_name { + ALGORITHM_OPTIONS_SYSV => ( + format!( + "{} {}{}", + sum.parse::().unwrap(), + sz.div_ceil(options.output_bits), + if not_file { "" } else { " " } + ), + !not_file, + String::new(), ), - (ALGORITHM_OPTIONS_SYSV, false) => println!( - "{} {} {}", - sum.parse::().unwrap(), - div_ceil(sz, options.output_bits), - filename.display() + ALGORITHM_OPTIONS_BSD => ( + format!( + "{:0bsd_width$} {:bsd_width$}{}", + sum.parse::().unwrap(), + sz.div_ceil(options.output_bits), + if not_file { "" } else { " " } + ), + !not_file, + String::new(), ), - (ALGORITHM_OPTIONS_BSD, true) => println!( - "{:0bsd_width$} {:bsd_width$}", - sum.parse::().unwrap(), - div_ceil(sz, options.output_bits) + ALGORITHM_OPTIONS_CRC | ALGORITHM_OPTIONS_CRC32B => ( + format!("{sum} {sz}{}", if not_file { "" } else { " " }), + !not_file, + String::new(), ), - (ALGORITHM_OPTIONS_BSD, false) => println!( - "{:0bsd_width$} {:bsd_width$} {}", - sum.parse::().unwrap(), - div_ceil(sz, options.output_bits), - filename.display() - ), - (ALGORITHM_OPTIONS_CRC, true) => println!("{sum} {sz}"), - (ALGORITHM_OPTIONS_CRC, false) => println!("{sum} {sz} {}", filename.display()), - (ALGORITHM_OPTIONS_BLAKE2B, _) if !options.untagged => { - if let Some(length) = options.length { - // Multiply by 8 here, as we want to print the length in bits. - println!("BLAKE2b-{} ({}) = {sum}", length * 8, filename.display()); - } else { - println!("BLAKE2b ({}) = {sum}", filename.display()); - } + ALGORITHM_OPTIONS_BLAKE2B if options.tag => { + ( + if let Some(length) = options.length { + // Multiply by 8 here, as we want to print the length in bits. + format!("BLAKE2b-{} (", length * 8) + } else { + "BLAKE2b (".to_owned() + }, + true, + format!(") = {sum}"), + ) } _ => { - if options.untagged { - println!("{sum} {}", filename.display()); + if options.tag { + ( + format!("{} (", options.algo_name.to_ascii_uppercase()), + true, + format!(") = {sum}"), + ) } else { - println!( - "{} ({}) = {sum}", - options.algo_name.to_ascii_uppercase(), - filename.display() - ); + let prefix = if options.asterisk { "*" } else { " " }; + (format!("{sum} {prefix}"), true, String::new()) } } + }; + print!("{before_filename}"); + if should_print_filename { + // The filename might not be valid UTF-8, and filename.display() would mangle the names. + // Therefore, emit the bytes directly to stdout, without any attempt at encoding them. + let _dropped_result = stdout().write_all(os_str_as_bytes(filename.as_os_str())?); } + print!("{after_filename}{}", options.line_ending); } Ok(()) } -fn digest_read( - digest: &mut Box, - reader: &mut BufReader, - output_bits: usize, -) -> io::Result<(String, usize)> { - digest.reset(); - - // Read bytes from `reader` and write those bytes to `digest`. - // - // If `binary` is `false` and the operating system is Windows, then - // `DigestWriter` replaces "\r\n" with "\n" before it writes the - // bytes into `digest`. Otherwise, it just inserts the bytes as-is. - // - // In order to support replacing "\r\n", we must call `finalize()` - // in order to support the possibility that the last character read - // from the reader was "\r". (This character gets buffered by - // `DigestWriter` and only written if the following character is - // "\n". But when "\r" is the last character read, we need to force - // it to be written.) - let mut digest_writer = DigestWriter::new(digest, true); - let output_size = std::io::copy(reader, &mut digest_writer)? as usize; - digest_writer.finalize(); - - if digest.output_bits() > 0 { - Ok((digest.result_str(), output_size)) - } else { - // Assume it's SHAKE. result_str() doesn't work with shake (as of 8/30/2016) - let mut bytes = vec![0; (output_bits + 7) / 8]; - digest.hash_finalize(&mut bytes); - Ok((encode(bytes), output_size)) - } -} - mod options { pub const ALGORITHM: &str = "algorithm"; pub const FILE: &str = "file"; pub const UNTAGGED: &str = "untagged"; + pub const TAG: &str = "tag"; pub const LENGTH: &str = "length"; pub const RAW: &str = "raw"; + pub const BASE64: &str = "base64"; + pub const CHECK: &str = "check"; + pub const STRICT: &str = "strict"; + pub const TEXT: &str = "text"; + pub const BINARY: &str = "binary"; + pub const STATUS: &str = "status"; + pub const WARN: &str = "warn"; + pub const IGNORE_MISSING: &str = "ignore-missing"; + pub const QUIET: &str = "quiet"; + pub const ZERO: &str = "zero"; +} + +/*** + * cksum has a bunch of legacy behavior. + * We handle this in this function to make sure they are self contained + * and "easier" to understand + */ +fn handle_tag_text_binary_flags>( + args: impl Iterator, +) -> UResult<(bool, bool)> { + let mut tag = true; + let mut binary = false; + + // --binary, --tag and --untagged are tight together: none of them + // conflicts with each other but --tag will reset "binary" and set "tag". + + for arg in args { + let arg = arg.as_ref(); + if arg == "-b" || arg == "--binary" { + binary = true; + } else if arg == "--tag" { + tag = true; + binary = false; + } else if arg == "--untagged" { + tag = false; + } + } + + Ok((tag, !tag && binary)) } #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let matches = uu_app().try_get_matches_from(args)?; + let check = matches.get_flag(options::CHECK); + let algo_name: &str = match matches.get_one::(options::ALGORITHM) { Some(v) => v, - None => ALGORITHM_OPTIONS_CRC, + None => { + if check { + // if we are doing a --check, we should not default to crc + "" + } else { + ALGORITHM_OPTIONS_CRC + } + } }; let input_length = matches.get_one::(options::LENGTH); - let length = if let Some(length) = input_length { - match length.to_owned() { - 0 => None, - n if n % 8 != 0 => { - // GNU's implementation seem to use these quotation marks - // in their error messages, so we do the same. - uucore::show_error!("invalid length: \u{2018}{length}\u{2019}"); - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "length is not a multiple of 8", - ) - .into()); - } - n if n > 512 => { - uucore::show_error!("invalid length: \u{2018}{length}\u{2019}"); - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "maximum digest length for \u{2018}BLAKE2b\u{2019} is 512 bits", - ) - .into()); + let length = match input_length { + Some(length) => { + if algo_name == ALGORITHM_OPTIONS_BLAKE2B { + calculate_blake2b_length(*length)? + } else { + return Err(ChecksumError::LengthOnlyForBlake2b.into()); } - n => { - if algo_name != ALGORITHM_OPTIONS_BLAKE2B { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "--length is only supported with --algorithm=blake2b", - ) - .into()); - } + } + None => None, + }; - // Divide by 8, as our blake2b implementation expects bytes - // instead of bits. - Some(n / 8) - } + if ["bsd", "crc", "sysv", "crc32b"].contains(&algo_name) && check { + return Err(ChecksumError::AlgorithmNotSupportedWithCheck.into()); + } + + if check { + let text_flag = matches.get_flag(options::TEXT); + let binary_flag = matches.get_flag(options::BINARY); + let strict = matches.get_flag(options::STRICT); + let status = matches.get_flag(options::STATUS); + let warn = matches.get_flag(options::WARN); + let ignore_missing = matches.get_flag(options::IGNORE_MISSING); + let quiet = matches.get_flag(options::QUIET); + let tag = matches.get_flag(options::TAG); + + if tag || binary_flag || text_flag { + return Err(ChecksumError::BinaryTextConflict.into()); } + // Determine the appropriate algorithm option to pass + let algo_option = if algo_name.is_empty() { + None + } else { + Some(algo_name) + }; + + // Execute the checksum validation based on the presence of files or the use of stdin + + let files = matches.get_many::(options::FILE).map_or_else( + || iter::once(OsStr::new("-")).collect::>(), + |files| files.map(OsStr::new).collect::>(), + ); + + let verbose = ChecksumVerbose::new(status, quiet, warn); + + let opts = ChecksumOptions { + binary: binary_flag, + ignore_missing, + strict, + verbose, + }; + + return perform_checksum_validation(files.iter().copied(), algo_option, length, opts); + } + + let (tag, asterisk) = handle_tag_text_binary_flags(std::env::args_os())?; + + let algo = detect_algo(algo_name, length)?; + let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO)); + + let output_format = if matches.get_flag(options::RAW) { + OutputFormat::Raw + } else if matches.get_flag(options::BASE64) { + OutputFormat::Base64 } else { - None + OutputFormat::Hexadecimal }; - let (name, algo, bits) = detect_algo(algo_name, length); - let opts = Options { - algo_name: name, - digest: algo, - output_bits: bits, + algo_name: algo.name, + digest: (algo.create_fn)(), + output_bits: algo.bits, length, - untagged: matches.get_flag(options::UNTAGGED), - raw: matches.get_flag(options::RAW), + tag, + output_format, + asterisk, + line_ending, }; - match matches.get_many::(options::FILE) { + match matches.get_many::(options::FILE) { Some(files) => cksum(opts, files.map(OsStr::new))?, None => cksum(opts, iter::once(OsStr::new("-")))?, }; @@ -361,14 +342,16 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) + .args_override_self(true) .arg( Arg::new(options::FILE) .hide(true) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) + .value_parser(ValueParser::os_string()) .value_hint(clap::ValueHint::FilePath), ) .arg( @@ -377,39 +360,130 @@ pub fn uu_app() -> Command { .short('a') .help("select the digest type to use. See DIGEST below") .value_name("ALGORITHM") - .value_parser([ - ALGORITHM_OPTIONS_SYSV, - ALGORITHM_OPTIONS_BSD, - ALGORITHM_OPTIONS_CRC, - ALGORITHM_OPTIONS_MD5, - ALGORITHM_OPTIONS_SHA1, - ALGORITHM_OPTIONS_SHA224, - ALGORITHM_OPTIONS_SHA256, - ALGORITHM_OPTIONS_SHA384, - ALGORITHM_OPTIONS_SHA512, - ALGORITHM_OPTIONS_BLAKE2B, - ALGORITHM_OPTIONS_SM3, - ]), + .value_parser(SUPPORTED_ALGORITHMS), ) .arg( Arg::new(options::UNTAGGED) .long(options::UNTAGGED) .help("create a reversed style checksum, without digest type") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with(options::TAG), + ) + .arg( + Arg::new(options::TAG) + .long(options::TAG) + .help("create a BSD style checksum, undo --untagged (default)") + .action(ArgAction::SetTrue) + .overrides_with(options::UNTAGGED), ) .arg( Arg::new(options::LENGTH) .long(options::LENGTH) .value_parser(value_parser!(usize)) .short('l') - .help("digest length in bits; must not exceed the max for the blake2 algorithm and must be a multiple of 8") + .help( + "digest length in bits; must not exceed the max for the blake2 algorithm \ + and must be a multiple of 8", + ) .action(ArgAction::Set), ) .arg( Arg::new(options::RAW) - .long(options::RAW) - .help("emit a raw binary digest, not hexadecimal") - .action(ArgAction::SetTrue), + .long(options::RAW) + .help("emit a raw binary digest, not hexadecimal") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::STRICT) + .long(options::STRICT) + .help("exit non-zero for improperly formatted checksum lines") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::CHECK) + .short('c') + .long(options::CHECK) + .help("read hashsums from the FILEs and check them") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::BASE64) + .long(options::BASE64) + .help("emit a base64 digest, not hexadecimal") + .action(ArgAction::SetTrue) + // Even though this could easily just override an earlier '--raw', + // GNU cksum does not permit these flags to be combined: + .conflicts_with(options::RAW), + ) + .arg( + Arg::new(options::TEXT) + .long(options::TEXT) + .short('t') + .hide(true) + .overrides_with(options::BINARY) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::BINARY) + .long(options::BINARY) + .short('b') + .hide(true) + .overrides_with(options::TEXT) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::WARN) + .short('w') + .long("warn") + .help("warn about improperly formatted checksum lines") + .action(ArgAction::SetTrue) + .overrides_with_all([options::STATUS, options::QUIET]), + ) + .arg( + Arg::new(options::STATUS) + .long("status") + .help("don't output anything, status code shows success") + .action(ArgAction::SetTrue) + .overrides_with_all([options::WARN, options::QUIET]), + ) + .arg( + Arg::new(options::QUIET) + .long(options::QUIET) + .help("don't print OK for each successfully verified file") + .action(ArgAction::SetTrue) + .overrides_with_all([options::WARN, options::STATUS]), + ) + .arg( + Arg::new(options::IGNORE_MISSING) + .long(options::IGNORE_MISSING) + .help("don't fail or report status for missing files") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::ZERO) + .long(options::ZERO) + .short('z') + .help( + "end each output line with NUL, not newline,\n and disable file name escaping", + ) + .action(ArgAction::SetTrue), ) .after_help(AFTER_HELP) } + +#[cfg(test)] +mod tests { + use crate::calculate_blake2b_length; + + #[test] + fn test_calculate_length() { + assert_eq!(calculate_blake2b_length(256).unwrap(), Some(32)); + assert_eq!(calculate_blake2b_length(512).unwrap(), None); + assert_eq!(calculate_blake2b_length(256).unwrap(), Some(32)); + calculate_blake2b_length(255).unwrap_err(); + + calculate_blake2b_length(33).unwrap_err(); + + calculate_blake2b_length(513).unwrap_err(); + } +} diff --git a/src/uu/comm/Cargo.toml b/src/uu/comm/Cargo.toml index cd759aad246..71617428039 100644 --- a/src/uu/comm/Cargo.toml +++ b/src/uu/comm/Cargo.toml @@ -1,22 +1,25 @@ [package] name = "uu_comm" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "comm ~ (uutils) compare sorted inputs" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/comm" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/comm.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["fs"] } [[bin]] name = "comm" diff --git a/src/uu/comm/src/comm.rs b/src/uu/comm/src/comm.rs index dd49ef53b02..11752c331a5 100644 --- a/src/uu/comm/src/comm.rs +++ b/src/uu/comm/src/comm.rs @@ -3,17 +3,17 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) delim mkdelim +// spell-checker:ignore (ToDO) delim mkdelim pairable use std::cmp::Ordering; -use std::fs::File; -use std::io::{self, stdin, BufRead, BufReader, Stdin}; -use std::path::Path; -use uucore::error::{FromIo, UResult}; +use std::fs::{File, metadata}; +use std::io::{self, BufRead, BufReader, Read, Stdin, stdin}; +use uucore::error::{FromIo, UResult, USimpleError}; +use uucore::fs::paths_refer_to_same_file; use uucore::line_ending::LineEnding; use uucore::{format_usage, help_about, help_usage}; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; const ABOUT: &str = help_about!("comm.md"); const USAGE: &str = help_usage!("comm.md"); @@ -28,6 +28,30 @@ mod options { pub const FILE_2: &str = "FILE2"; pub const TOTAL: &str = "total"; pub const ZERO_TERMINATED: &str = "zero-terminated"; + pub const CHECK_ORDER: &str = "check-order"; + pub const NO_CHECK_ORDER: &str = "nocheck-order"; +} + +#[derive(Debug, Clone, Copy)] +enum FileNumber { + One, + Two, +} + +impl FileNumber { + fn as_str(&self) -> &'static str { + match self { + FileNumber::One => "1", + FileNumber::Two => "2", + } + } +} + +struct OrderChecker { + last_line: Vec, + file_num: FileNumber, + check_order: bool, + has_error: bool, } enum Input { @@ -42,7 +66,7 @@ struct LineReader { impl LineReader { fn new(input: Input, line_ending: LineEnding) -> Self { - Self { input, line_ending } + Self { line_ending, input } } fn read_line(&mut self, buf: &mut Vec) -> io::Result { @@ -61,12 +85,74 @@ impl LineReader { } } -fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { - let delim = match opts.get_one::(options::DELIMITER).unwrap().as_str() { - "" => "\0", - delim => delim, - }; +impl OrderChecker { + fn new(file_num: FileNumber, check_order: bool) -> Self { + Self { + last_line: Vec::new(), + file_num, + check_order, + has_error: false, + } + } + + fn verify_order(&mut self, current_line: &[u8]) -> bool { + if self.last_line.is_empty() { + self.last_line = current_line.to_vec(); + return true; + } + let is_ordered = current_line >= &self.last_line; + if !is_ordered && !self.has_error { + eprintln!( + "comm: file {} is not in sorted order", + self.file_num.as_str() + ); + self.has_error = true; + } + + self.last_line = current_line.to_vec(); + is_ordered || !self.check_order + } +} + +// Check if two files are identical by comparing their contents +pub fn are_files_identical(path1: &str, path2: &str) -> io::Result { + // First compare file sizes + let metadata1 = metadata(path1)?; + let metadata2 = metadata(path2)?; + + if metadata1.len() != metadata2.len() { + return Ok(false); + } + + let file1 = File::open(path1)?; + let file2 = File::open(path2)?; + + let mut reader1 = BufReader::new(file1); + let mut reader2 = BufReader::new(file2); + + let mut buffer1 = [0; 8192]; + let mut buffer2 = [0; 8192]; + + loop { + let bytes1 = reader1.read(&mut buffer1)?; + let bytes2 = reader2.read(&mut buffer2)?; + + if bytes1 != bytes2 { + return Ok(false); + } + + if bytes1 == 0 { + return Ok(true); + } + + if buffer1[..bytes1] != buffer2[..bytes2] { + return Ok(false); + } + } +} + +fn comm(a: &mut LineReader, b: &mut LineReader, delim: &str, opts: &ArgMatches) -> UResult<()> { let width_col_1 = usize::from(!opts.get_flag(options::COLUMN_1)); let width_col_2 = usize::from(!opts.get_flag(options::COLUMN_2)); @@ -82,6 +168,26 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { let mut total_col_2 = 0; let mut total_col_3 = 0; + let check_order = opts.get_flag(options::CHECK_ORDER); + let no_check_order = opts.get_flag(options::NO_CHECK_ORDER); + + // Determine if we should perform order checking + let should_check_order = !no_check_order + && (check_order + || if let (Some(file1), Some(file2)) = ( + opts.get_one::(options::FILE_1), + opts.get_one::(options::FILE_2), + ) { + !(paths_refer_to_same_file(file1, file2, true) + || are_files_identical(file1, file2).unwrap_or(false)) + } else { + true + }); + + let mut checker1 = OrderChecker::new(FileNumber::One, check_order); + let mut checker2 = OrderChecker::new(FileNumber::Two, check_order); + let mut input_error = false; + while na.is_ok() || nb.is_ok() { let ord = match (na.is_ok(), nb.is_ok()) { (false, true) => Ordering::Greater, @@ -97,6 +203,9 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { match ord { Ordering::Less => { + if should_check_order && !checker1.verify_order(ra) { + break; + } if !opts.get_flag(options::COLUMN_1) { print!("{}", String::from_utf8_lossy(ra)); } @@ -105,6 +214,9 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { total_col_1 += 1; } Ordering::Greater => { + if should_check_order && !checker2.verify_order(rb) { + break; + } if !opts.get_flag(options::COLUMN_2) { print!("{delim_col_2}{}", String::from_utf8_lossy(rb)); } @@ -113,6 +225,10 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { total_col_2 += 1; } Ordering::Equal => { + if should_check_order && (!checker1.verify_order(ra) || !checker2.verify_order(rb)) + { + break; + } if !opts.get_flag(options::COLUMN_3) { print!("{delim_col_3}{}", String::from_utf8_lossy(ra)); } @@ -123,19 +239,37 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { total_col_3 += 1; } } + + // Track if we've seen any order errors + if (checker1.has_error || checker2.has_error) && !input_error && !check_order { + input_error = true; + } } if opts.get_flag(options::TOTAL) { let line_ending = LineEnding::from_zero_flag(opts.get_flag(options::ZERO_TERMINATED)); print!("{total_col_1}{delim}{total_col_2}{delim}{total_col_3}{delim}total{line_ending}"); } + + if should_check_order && (checker1.has_error || checker2.has_error) { + // Print the input error message once at the end + if input_error { + eprintln!("comm: input is not in sorted order"); + } + Err(USimpleError::new(1, "")) + } else { + Ok(()) + } } fn open_file(name: &str, line_ending: LineEnding) -> io::Result { if name == "-" { Ok(LineReader::new(Input::Stdin(stdin()), line_ending)) } else { - let f = File::open(Path::new(name))?; + if metadata(name)?.is_dir() { + return Err(io::Error::other("Is a directory")); + } + let f = File::open(name)?; Ok(LineReader::new( Input::FileIn(BufReader::new(f)), line_ending, @@ -152,16 +286,38 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let mut f1 = open_file(filename1, line_ending).map_err_context(|| filename1.to_string())?; let mut f2 = open_file(filename2, line_ending).map_err_context(|| filename2.to_string())?; - comm(&mut f1, &mut f2, &matches); - Ok(()) + // Due to default_value(), there must be at least one value here, thus unwrap() must not panic. + let all_delimiters = matches + .get_many::(options::DELIMITER) + .unwrap() + .map(String::from) + .collect::>(); + for delim in &all_delimiters[1..] { + // Note that this check is very different from ".conflicts_with_self(true).action(ArgAction::Set)", + // as this accepts duplicate *identical* arguments. + if delim != &all_delimiters[0] { + // Note: This intentionally deviate from the GNU error message by inserting the word "conflicting". + return Err(USimpleError::new( + 1, + "multiple conflicting output delimiters specified", + )); + } + } + let delim = match &*all_delimiters[0] { + "" => "\0", + delim => delim, + }; + + comm(&mut f1, &mut f2, delim, &matches) } pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) + .args_override_self(true) .arg( Arg::new(options::COLUMN_1) .short('1') @@ -186,6 +342,8 @@ pub fn uu_app() -> Command { .help("separate columns with STR") .value_name("STR") .default_value(options::DELIMITER_DEFAULT) + .allow_hyphen_values(true) + .action(ArgAction::Append) .hide_default_value(true), ) .arg( @@ -212,4 +370,17 @@ pub fn uu_app() -> Command { .help("output a summary") .action(ArgAction::SetTrue), ) + .arg( + Arg::new(options::CHECK_ORDER) + .long(options::CHECK_ORDER) + .help("check that the input is correctly sorted, even if all input lines are pairable") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::NO_CHECK_ORDER) + .long(options::NO_CHECK_ORDER) + .help("do not check that the input is correctly sorted") + .action(ArgAction::SetTrue) + .conflicts_with(options::CHECK_ORDER), + ) } diff --git a/src/uu/cp/Cargo.toml b/src/uu/cp/Cargo.toml index 7db6b202bb2..fd5b4696e03 100644 --- a/src/uu/cp/Cargo.toml +++ b/src/uu/cp/Cargo.toml @@ -1,19 +1,18 @@ [package] name = "uu_cp" -version = "0.0.24" -authors = [ - "Jordy Dickinson ", - "Joshua S. Miller ", - "uutils developers", -] -license = "MIT" description = "cp ~ (uutils) copy SOURCE to DESTINATION" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/cp" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/cp.rs" @@ -22,12 +21,16 @@ path = "src/cp.rs" clap = { workspace = true } filetime = { workspace = true } libc = { workspace = true } +linux-raw-sys = { workspace = true } quick-error = { workspace = true } selinux = { workspace = true, optional = true } uucore = { workspace = true, features = [ "backup-control", + "buf-copy", "entries", "fs", + "fsxattr", + "parser", "perms", "mode", "update-control", @@ -44,5 +47,5 @@ name = "cp" path = "src/main.rs" [features] -feat_selinux = ["selinux"] +feat_selinux = ["selinux", "uucore/selinux"] feat_acl = ["exacl"] diff --git a/src/uu/cp/src/copydir.rs b/src/uu/cp/src/copydir.rs index 7a9d797e81c..d2e367c5c19 100644 --- a/src/uu/cp/src/copydir.rs +++ b/src/uu/cp/src/copydir.rs @@ -2,7 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore TODO canonicalizes direntry pathbuf symlinked +// spell-checker:ignore TODO canonicalizes direntry pathbuf symlinked IRWXO IRWXG //! Recursively copy the contents of a directory. //! //! See the [`copy_directory`] function for more information. @@ -18,7 +18,7 @@ use indicatif::ProgressBar; use uucore::display::Quotable; use uucore::error::UIoError; use uucore::fs::{ - canonicalize, path_ends_with_terminator, FileInformation, MissingHandling, ResolveMode, + FileInformation, MissingHandling, ResolveMode, canonicalize, path_ends_with_terminator, }; use uucore::show; use uucore::show_error; @@ -26,8 +26,8 @@ use uucore::uio_error; use walkdir::{DirEntry, WalkDir}; use crate::{ - aligned_ancestors, context_for, copy_attributes, copy_file, copy_link, CopyResult, Error, - Options, + CopyResult, Error, Options, aligned_ancestors, context_for, copy_attributes, copy_file, + copy_link, }; /// Ensure a Windows path starts with a `\\?`. @@ -42,8 +42,9 @@ fn adjust_canonicalization(p: &Path) -> Cow { .components() .next() .and_then(|comp| comp.as_os_str().to_str()) - .map(|p_str| p_str.starts_with(VERBATIM_PREFIX) || p_str.starts_with(DEVICE_NS_PREFIX)) - .unwrap_or_default(); + .is_some_and(|p_str| { + p_str.starts_with(VERBATIM_PREFIX) || p_str.starts_with(DEVICE_NS_PREFIX) + }); if has_prefix { p.into() @@ -79,6 +80,12 @@ fn get_local_to_root_parent( } } +/// Given an iterator, return all its items except the last. +fn skip_last(mut iter: impl Iterator) -> impl Iterator { + let last = iter.next(); + iter.scan(last, |state, item| state.replace(item)) +} + /// Paths that are invariant throughout the traversal when copying a directory. struct Context<'a> { /// The current working directory at the time of starting the traversal. @@ -95,7 +102,7 @@ struct Context<'a> { } impl<'a> Context<'a> { - fn new(root: &'a Path, target: &'a Path) -> std::io::Result { + fn new(root: &'a Path, target: &'a Path) -> io::Result { let current_dir = env::current_dir()?; let root_path = current_dir.join(root); let root_parent = if target.exists() && !root.to_str().unwrap().ends_with("/.") { @@ -162,20 +169,21 @@ struct Entry { } impl Entry { - fn new( + fn new>( context: &Context, - direntry: &DirEntry, + source: A, no_target_dir: bool, ) -> Result { - let source_relative = direntry.path().to_path_buf(); + let source = source.as_ref(); + let source_relative = source.to_path_buf(); let source_absolute = context.current_dir.join(&source_relative); let mut descendant = get_local_to_root_parent(&source_absolute, context.root_parent.as_deref())?; if no_target_dir { - let source_is_dir = direntry.path().is_dir(); + let source_is_dir = source.is_dir(); if path_ends_with_terminator(context.target) && source_is_dir { - if let Err(e) = std::fs::create_dir_all(context.target) { - eprintln!("Failed to create directory: {}", e); + if let Err(e) = fs::create_dir_all(context.target) { + eprintln!("Failed to create directory: {e}"); } } else { descendant = descendant.strip_prefix(context.root)?.to_path_buf(); @@ -193,34 +201,15 @@ impl Entry { } } -/// Decide whether the given path ends with `/.`. -/// -/// # Examples -/// -/// ```rust,ignore -/// assert!(ends_with_slash_dot("/.")); -/// assert!(ends_with_slash_dot("./.")); -/// assert!(ends_with_slash_dot("a/.")); -/// -/// assert!(!ends_with_slash_dot(".")); -/// assert!(!ends_with_slash_dot("./")); -/// assert!(!ends_with_slash_dot("a/..")); -/// ``` -fn ends_with_slash_dot

(path: P) -> bool -where - P: AsRef, -{ - // `path.ends_with(".")` does not seem to work - path.as_ref().display().to_string().ends_with("/.") -} - +#[allow(clippy::too_many_arguments)] /// Copy a single entry during a directory traversal. fn copy_direntry( - progress_bar: &Option, + progress_bar: Option<&ProgressBar>, entry: Entry, options: &Options, symlinked_files: &mut HashSet, preserve_hard_links: bool, + copied_destinations: &HashSet, copied_files: &mut HashMap, ) -> CopyResult<()> { let Entry { @@ -238,77 +227,54 @@ fn copy_direntry( // If the source is a directory and the destination does not // exist, ... - if source_absolute.is_dir() - && !ends_with_slash_dot(&source_absolute) - && !local_to_target.exists() - { - if target_is_file { - return Err("cannot overwrite non-directory with directory".into()); + if source_absolute.is_dir() && !local_to_target.exists() { + return if target_is_file { + Err("cannot overwrite non-directory with directory".into()) } else { - // TODO Since the calling code is traversing from the root - // of the directory structure, I don't think - // `create_dir_all()` will have any benefit over - // `create_dir()`, since all the ancestor directories - // should have already been created. - fs::create_dir_all(&local_to_target)?; + build_dir(&local_to_target, false, options, Some(&source_absolute))?; if options.verbose { println!("{}", context_for(&source_relative, &local_to_target)); } - return Ok(()); - } + Ok(()) + }; } // If the source is not a directory, then we need to copy the file. if !source_absolute.is_dir() { - if preserve_hard_links { - match copy_file( - progress_bar, - &source_absolute, - local_to_target.as_path(), - options, - symlinked_files, - copied_files, - false, - ) { - Ok(_) => Ok(()), - Err(err) => { - if source_absolute.is_symlink() { - // silent the error with a symlink - // In case we do --archive, we might copy the symlink - // before the file itself - Ok(()) - } else { - Err(err) - } + if let Err(err) = copy_file( + progress_bar, + &source_absolute, + local_to_target.as_path(), + options, + symlinked_files, + copied_destinations, + copied_files, + false, + ) { + if preserve_hard_links { + if !source_absolute.is_symlink() { + return Err(err); } - }?; - } else { - // At this point, `path` is just a plain old file. - // Terminate this function immediately if there is any - // kind of error *except* a "permission denied" error. - // - // TODO What other kinds of errors, if any, should - // cause us to continue walking the directory? - match copy_file( - progress_bar, - &source_absolute, - local_to_target.as_path(), - options, - symlinked_files, - copied_files, - false, - ) { - Ok(_) => {} - Err(Error::IoErrContext(e, _)) - if e.kind() == std::io::ErrorKind::PermissionDenied => - { - show!(uio_error!( - e, - "cannot open {} for reading", - source_relative.quote(), - )); + // silent the error with a symlink + // In case we do --archive, we might copy the symlink + // before the file itself + } else { + // At this point, `path` is just a plain old file. + // Terminate this function immediately if there is any + // kind of error *except* a "permission denied" error. + // + // TODO What other kinds of errors, if any, should + // cause us to continue walking the directory? + match err { + Error::IoErrContext(e, _) if e.kind() == io::ErrorKind::PermissionDenied => { + show!(uio_error!( + e, + "cannot open {} for reading", + source_relative.quote(), + )); + } + e => return Err(e), } - Err(e) => return Err(e), } } } @@ -323,19 +289,17 @@ fn copy_direntry( /// /// Any errors encountered copying files in the tree will be logged but /// will not cause a short-circuit. +#[allow(clippy::too_many_arguments)] pub(crate) fn copy_directory( - progress_bar: &Option, + progress_bar: Option<&ProgressBar>, root: &Path, target: &Path, options: &Options, symlinked_files: &mut HashSet, + copied_destinations: &HashSet, copied_files: &mut HashMap, source_in_command_line: bool, ) -> CopyResult<()> { - if !options.recursive { - return Err(format!("-r not specified; omitting directory {}", root.quote()).into()); - } - // if no-dereference is enabled and this is a symlink, copy it as a file if !options.dereference(source_in_command_line) && root.is_symlink() { return copy_file( @@ -344,11 +308,16 @@ pub(crate) fn copy_directory( target, options, symlinked_files, + copied_destinations, copied_files, source_in_command_line, ); } + if !options.recursive { + return Err(format!("-r not specified; omitting directory {}", root.quote()).into()); + } + // check if root is a prefix of target if path_has_prefix(target, root)? { return Err(format!( @@ -370,8 +339,7 @@ pub(crate) fn copy_directory( let tmp = if options.parents { if let Some(parent) = root.parent() { let new_target = target.join(parent); - std::fs::create_dir_all(&new_target)?; - + build_dir(&new_target, true, options, None)?; if options.verbose { // For example, if copying file `a/b/c` and its parents // to directory `d/`, then print @@ -403,6 +371,9 @@ pub(crate) fn copy_directory( Err(e) => return Err(format!("failed to get current directory {e}").into()), }; + // The directory we were in during the previous iteration + let mut last_iter: Option = None; + // Traverse the contents of the directory, copying each one. for direntry_result in WalkDir::new(root) .same_file_system(options.one_file_system) @@ -410,30 +381,104 @@ pub(crate) fn copy_directory( { match direntry_result { Ok(direntry) => { - let entry = Entry::new(&context, &direntry, options.no_target_dir)?; + let entry = Entry::new(&context, direntry.path(), options.no_target_dir)?; + copy_direntry( progress_bar, entry, options, symlinked_files, preserve_hard_links, + copied_destinations, copied_files, )?; + + // We omit certain permissions when creating directories + // to prevent other users from accessing them before they're done. + // We thus need to fix the permissions of each directory we copy + // once it's contents are ready. + // This "fixup" is implemented here in a memory-efficient manner. + // + // We detect iterations where we "walk up" the directory tree, + // and fix permissions on all the directories we exited. + // (Note that there can be more than one! We might step out of + // `./a/b/c` into `./a/`, in which case we'll need to fix the + // permissions of both `./a/b/c` and `./a/b`, in that order.) + if direntry.file_type().is_dir() { + // If true, last_iter is not a parent of this iter. + // The means we just exited a directory. + let went_up = if let Some(last_iter) = &last_iter { + last_iter.path().strip_prefix(direntry.path()).is_ok() + } else { + false + }; + + if went_up { + // Compute the "difference" between `last_iter` and `direntry`. + // For example, if... + // - last_iter = `a/b/c/d` + // - direntry = `a/b` + // then diff = `c/d` + // + // All the unwraps() here are unreachable. + let last_iter = last_iter.as_ref().unwrap(); + let diff = last_iter.path().strip_prefix(direntry.path()).unwrap(); + + // Fix permissions for every entry in `diff`, inside-out. + // We skip the last directory (which will be `.`) because + // its permissions will be fixed when we walk _out_ of it. + // (at this point, we might not be done copying `.`!) + for p in skip_last(diff.ancestors()) { + let src = direntry.path().join(p); + let entry = Entry::new(&context, &src, options.no_target_dir)?; + + copy_attributes( + &entry.source_absolute, + &entry.local_to_target, + &options.attributes, + )?; + } + } + + last_iter = Some(direntry); + } } + // Print an error message, but continue traversing the directory. - Err(e) => show_error!("{}", e), + Err(e) => show_error!("{e}"), + } + } + + // Handle final directory permission fixes. + // This is almost the same as the permission-fixing code above, + // with minor differences (commented) + if let Some(last_iter) = last_iter { + let diff = last_iter.path().strip_prefix(root).unwrap(); + + // Do _not_ skip `.` this time, since we know we're done. + // This is where we fix the permissions of the top-level + // directory we just copied. + for p in diff.ancestors() { + let src = root.join(p); + let entry = Entry::new(&context, &src, options.no_target_dir)?; + + copy_attributes( + &entry.source_absolute, + &entry.local_to_target, + &options.attributes, + )?; } } - // Copy the attributes from the root directory to the target directory. + // Also fix permissions for parent directories, + // if we were asked to create them. if options.parents { let dest = target.join(root.file_name().unwrap()); - copy_attributes(root, dest.as_path(), &options.attributes)?; for (x, y) in aligned_ancestors(root, dest.as_path()) { - copy_attributes(x, y, &options.attributes)?; + if let Ok(src) = canonicalize(x, MissingHandling::Normal, ResolveMode::Physical) { + copy_attributes(&src, y, &options.attributes)?; + } } - } else { - copy_attributes(root, target, &options.attributes)?; } Ok(()) @@ -463,25 +508,61 @@ pub fn path_has_prefix(p1: &Path, p2: &Path) -> io::Result { Ok(pathbuf1.starts_with(pathbuf2)) } -#[cfg(test)] -mod tests { - use super::ends_with_slash_dot; - - #[test] - #[allow(clippy::cognitive_complexity)] - fn test_ends_with_slash_dot() { - assert!(ends_with_slash_dot("/.")); - assert!(ends_with_slash_dot("./.")); - assert!(ends_with_slash_dot("../.")); - assert!(ends_with_slash_dot("a/.")); - assert!(ends_with_slash_dot("/a/.")); - - assert!(!ends_with_slash_dot("")); - assert!(!ends_with_slash_dot(".")); - assert!(!ends_with_slash_dot("./")); - assert!(!ends_with_slash_dot("..")); - assert!(!ends_with_slash_dot("/..")); - assert!(!ends_with_slash_dot("a/..")); - assert!(!ends_with_slash_dot("/a/..")); +/// Builds a directory at the specified path with the given options. +/// +/// # Notes +/// - If `copy_attributes_from` is `Some`, the new directory's attributes will be +/// copied from the provided file. Otherwise, the new directory will have the default +/// attributes for the current user. +/// - This method excludes certain permissions if ownership or special mode bits could +/// potentially change. (See `test_dir_perm_race_with_preserve_mode_and_ownership``) +/// - The `recursive` flag determines whether parent directories should be created +/// if they do not already exist. +// we need to allow unused_variable since `options` might be unused in non unix systems +#[allow(unused_variables)] +fn build_dir( + path: &PathBuf, + recursive: bool, + options: &Options, + copy_attributes_from: Option<&Path>, +) -> CopyResult<()> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(recursive); + + // To prevent unauthorized access before the folder is ready, + // exclude certain permissions if ownership or special mode bits + // could potentially change. + #[cfg(unix)] + { + use crate::Preserve; + use std::os::unix::fs::PermissionsExt; + + // we need to allow trivial casts here because some systems like linux have u32 constants in + // in libc while others don't. + #[allow(clippy::unnecessary_cast)] + let mut excluded_perms = if matches!(options.attributes.ownership, Preserve::Yes { .. }) { + libc::S_IRWXG | libc::S_IRWXO // exclude rwx for group and other + } else if matches!(options.attributes.mode, Preserve::Yes { .. }) { + libc::S_IWGRP | libc::S_IWOTH //exclude w for group and other + } else { + 0 + } as u32; + + let umask = if copy_attributes_from.is_some() + && matches!(options.attributes.mode, Preserve::Yes { .. }) + { + !fs::symlink_metadata(copy_attributes_from.unwrap())? + .permissions() + .mode() + } else { + uucore::mode::get_umask() + }; + + excluded_perms |= umask; + let mode = !excluded_perms & 0o777; //use only the last three octet bits + std::os::unix::fs::DirBuilderExt::mode(&mut builder, mode); } + + builder.create(path)?; + Ok(()) } diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index 28e9b678471..203e7836feb 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -3,45 +3,41 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. // spell-checker:ignore (ToDO) copydir ficlone fiemap ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs xattrs symlinked deduplicated advcpmv nushell IRWXG IRWXO IRWXU IRWXUGO IRWXU IRWXG IRWXO IRWXUGO -#![allow(clippy::missing_safety_doc)] -#![allow(clippy::extra_unused_lifetimes)] use quick_error::quick_error; use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; -use std::env; -#[cfg(not(windows))] -use std::ffi::CString; -use std::fs::{self, File, OpenOptions}; +use std::ffi::OsString; +use std::fs::{self, Metadata, OpenOptions, Permissions}; use std::io; #[cfg(unix)] -use std::os::unix::ffi::OsStrExt; -#[cfg(unix)] use std::os::unix::fs::{FileTypeExt, PermissionsExt}; use std::path::{Path, PathBuf, StripPrefixError}; -use std::string::ToString; +#[cfg(all(unix, not(target_os = "android")))] +use uucore::fsxattr::copy_xattrs; -use clap::{builder::ValueParser, crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command, builder::ValueParser, value_parser}; use filetime::FileTime; use indicatif::{ProgressBar, ProgressStyle}; -#[cfg(unix)] -use libc::mkfifo; use quick_error::ResultExt; use platform::copy_on_write; use uucore::display::Quotable; -use uucore::error::{set_exit_code, UClapError, UError, UResult, UUsageError}; +use uucore::error::{UClapError, UError, UResult, UUsageError, set_exit_code}; +#[cfg(unix)] +use uucore::fs::make_fifo; use uucore::fs::{ - are_hardlinks_to_same_file, canonicalize, is_symlink_loop, path_ends_with_terminator, - paths_refer_to_same_file, FileInformation, MissingHandling, ResolveMode, + FileInformation, MissingHandling, ResolveMode, are_hardlinks_to_same_file, canonicalize, + get_filename, is_symlink_loop, normalize_path, path_ends_with_terminator, + paths_refer_to_same_file, }; use uucore::{backup_control, update_control}; // These are exposed for projects (e.g. nushell) that want to create an `Options` value, which // requires these enum. pub use uucore::{backup_control::BackupMode, update_control::UpdateMode}; use uucore::{ - format_usage, help_about, help_section, help_usage, prompt_yes, show_error, show_warning, - util_name, + format_usage, help_about, help_section, help_usage, + parser::shortcut_value_parser::ShortcutValueParser, prompt_yes, show_error, show_warning, }; use crate::copydir::copy_directory; @@ -53,11 +49,11 @@ quick_error! { #[derive(Debug)] pub enum Error { /// Simple io::Error wrapper - IoErr(err: io::Error) { from() source(err) display("{}", err)} + IoErr(err: io::Error) { from() source(err) display("{err}")} /// Wrapper for io::Error with path context IoErrContext(err: io::Error, path: String) { - display("{}: {}", path, err) + display("{path}: {err}") context(path: &'a str, err: io::Error) -> (err, path.to_owned()) context(context: String, err: io::Error) -> (err, context) source(err) @@ -65,7 +61,7 @@ quick_error! { /// General copy error Error(err: String) { - display("{}", err) + display("{err}") from(err: String) -> (err) from(err: &'static str) -> (err.to_string()) } @@ -75,25 +71,27 @@ quick_error! { NotAllFilesCopied {} /// Simple walkdir::Error wrapper - WalkDirErr(err: walkdir::Error) { from() display("{}", err) source(err) } + WalkDirErr(err: walkdir::Error) { from() display("{err}") source(err) } /// Simple std::path::StripPrefixError wrapper StripPrefixError(err: StripPrefixError) { from() } /// Result of a skipped file - /// Currently happens when "no" is selected in interactive mode - Skipped { } + /// Currently happens when "no" is selected in interactive mode or when + /// `no-clobber` flag is set and destination is already present. + /// `exit with error` is used to determine which exit code should be returned. + Skipped(exit_with_error:bool) { } /// Result of a skipped file - InvalidArgument(description: String) { display("{}", description) } + InvalidArgument(description: String) { display("{description}") } /// All standard options are included as an an implementation /// path, but those that are not implemented yet should return /// a NotImplemented error. - NotImplemented(opt: String) { display("Option '{}' not yet implemented.", opt) } + NotImplemented(opt: String) { display("Option '{opt}' not yet implemented.") } /// Invalid arguments to backup - Backup(description: String) { display("{}\nTry '{} --help' for more information.", description, uucore::execution_phrase()) } + Backup(description: String) { display("{description}\nTry '{} --help' for more information.", uucore::execution_phrase()) } NotADirectory(path: PathBuf) { display("'{}' is not a directory", path.display()) } } @@ -108,15 +106,16 @@ impl UError for Error { pub type CopyResult = Result; /// Specifies how to overwrite files. -#[derive(Clone, Copy, Eq, PartialEq)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Default)] pub enum ClobberMode { Force, RemoveDestination, + #[default] Standard, } /// Specifies whether files should be overwritten. -#[derive(Clone, Copy, Eq, PartialEq)] +#[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum OverwriteMode { /// [Default] Always overwrite existing files Clobber(ClobberMode), @@ -126,18 +125,39 @@ pub enum OverwriteMode { NoClobber, } +impl Default for OverwriteMode { + fn default() -> Self { + Self::Clobber(ClobberMode::default()) + } +} + /// Possible arguments for `--reflink`. -#[derive(Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum ReflinkMode { Always, Auto, Never, } +impl Default for ReflinkMode { + #[allow(clippy::derivable_impls)] + fn default() -> Self { + #[cfg(any(target_os = "linux", target_os = "android", target_os = "macos"))] + { + ReflinkMode::Auto + } + #[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))] + { + ReflinkMode::Never + } + } +} + /// Possible arguments for `--sparse`. -#[derive(Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Default)] pub enum SparseMode { Always, + #[default] Auto, Never, } @@ -150,10 +170,11 @@ pub enum TargetType { } /// Copy action to perform -#[derive(PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Default)] pub enum CopyMode { Link, SymLink, + #[default] Copy, Update, AttrOnly, @@ -172,7 +193,7 @@ pub enum CopyMode { /// For full compatibility with GNU, these options should also combine. We /// currently only do a best effort imitation of that behavior, because it is /// difficult to achieve in clap, especially with `--no-preserve`. -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct Attributes { #[cfg(unix)] pub ownership: Preserve, @@ -183,6 +204,12 @@ pub struct Attributes { pub xattr: Preserve, } +impl Default for Attributes { + fn default() -> Self { + Self::NONE + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Preserve { // explicit means whether the --no-preserve flag is used or not to distinguish out the default value. @@ -222,6 +249,7 @@ impl Ord for Preserve { /// /// The fields are documented with the arguments that determine their value. #[allow(dead_code)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct Options { /// `--attributes-only` pub attributes_only: bool, @@ -283,6 +311,47 @@ pub struct Options { pub verbose: bool, /// `-g`, `--progress` pub progress_bar: bool, + /// -Z + pub set_selinux_context: bool, + // --context + pub context: Option, +} + +impl Default for Options { + fn default() -> Self { + Self { + attributes_only: false, + backup: BackupMode::default(), + copy_contents: false, + cli_dereference: false, + copy_mode: CopyMode::default(), + dereference: false, + no_target_dir: false, + one_file_system: false, + overwrite: OverwriteMode::default(), + parents: false, + sparse_mode: SparseMode::default(), + strip_trailing_slashes: false, + reflink_mode: ReflinkMode::default(), + attributes: Attributes::default(), + recursive: false, + backup_suffix: backup_control::DEFAULT_BACKUP_SUFFIX.to_owned(), + target_dir: None, + update: UpdateMode::default(), + debug: false, + verbose: false, + progress_bar: false, + set_selinux_context: false, + context: None, + } + } +} + +/// Enum representing if a file has been skipped. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PerformedAction { + Copied, + Skipped, } /// Enum representing various debug states of the offload and reflink actions. @@ -385,6 +454,7 @@ mod options { pub const RECURSIVE: &str = "recursive"; pub const REFLINK: &str = "reflink"; pub const REMOVE_DESTINATION: &str = "remove-destination"; + pub const SELINUX: &str = "Z"; pub const SPARSE: &str = "sparse"; pub const STRIP_TRAILING_SLASHES: &str = "strip-trailing-slashes"; pub const SYMBOLIC_LINK: &str = "symbolic-link"; @@ -399,22 +469,20 @@ static PRESERVABLE_ATTRIBUTES: &[&str] = &[ "ownership", "timestamps", "context", - "link", "links", "xattr", "all", ]; #[cfg(not(unix))] -static PRESERVABLE_ATTRIBUTES: &[&str] = &[ - "mode", - "timestamps", - "context", - "link", - "links", - "xattr", - "all", -]; +static PRESERVABLE_ATTRIBUTES: &[&str] = + &["mode", "timestamps", "context", "links", "xattr", "all"]; + +const PRESERVE_DEFAULT_VALUES: &str = if cfg!(unix) { + "mode,ownership,timestamp" +} else { + "mode,timestamp" +}; pub fn uu_app() -> Command { const MODE_ARGS: &[&str] = &[ @@ -425,7 +493,7 @@ pub fn uu_app() -> Command { options::COPY_CONTENTS, ]; Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .after_help(format!( @@ -478,8 +546,8 @@ pub fn uu_app() -> Command { ) .arg( Arg::new(options::RECURSIVE) - .short('r') - .visible_short_alias('R') + .short('R') + .visible_short_alias('r') .long(options::RECURSIVE) // --archive sets this option .help("copy directories recursively") @@ -546,7 +614,7 @@ pub fn uu_app() -> Command { .overrides_with_all(MODE_ARGS) .require_equals(true) .default_missing_value("always") - .value_parser(["auto", "always", "never"]) + .value_parser(ShortcutValueParser::new(["auto", "always", "never"])) .num_args(0..=1) .help("control clone/CoW copies. See below"), ) @@ -562,17 +630,11 @@ pub fn uu_app() -> Command { .long(options::PRESERVE) .action(ArgAction::Append) .use_value_delimiter(true) - .value_parser(clap::builder::PossibleValuesParser::new( - PRESERVABLE_ATTRIBUTES, - )) + .value_parser(ShortcutValueParser::new(PRESERVABLE_ATTRIBUTES)) .num_args(0..) .require_equals(true) .value_name("ATTR_LIST") - .overrides_with_all([ - options::ARCHIVE, - options::PRESERVE_DEFAULT_ATTRIBUTES, - options::NO_PRESERVE, - ]) + .default_missing_value(PRESERVE_DEFAULT_VALUES) // -d sets this option // --archive sets this option .help( @@ -584,19 +646,18 @@ pub fn uu_app() -> Command { Arg::new(options::PRESERVE_DEFAULT_ATTRIBUTES) .short('p') .long(options::PRESERVE_DEFAULT_ATTRIBUTES) - .overrides_with_all([options::PRESERVE, options::NO_PRESERVE, options::ARCHIVE]) .help("same as --preserve=mode,ownership(unix only),timestamps") .action(ArgAction::SetTrue), ) .arg( Arg::new(options::NO_PRESERVE) .long(options::NO_PRESERVE) + .action(ArgAction::Append) + .use_value_delimiter(true) + .value_parser(ShortcutValueParser::new(PRESERVABLE_ATTRIBUTES)) + .num_args(0..) + .require_equals(true) .value_name("ATTR_LIST") - .overrides_with_all([ - options::PRESERVE_DEFAULT_ATTRIBUTES, - options::PRESERVE, - options::ARCHIVE, - ]) .help("don't preserve the specified attributes"), ) .arg( @@ -633,11 +694,6 @@ pub fn uu_app() -> Command { Arg::new(options::ARCHIVE) .short('a') .long(options::ARCHIVE) - .overrides_with_all([ - options::PRESERVE_DEFAULT_ATTRIBUTES, - options::PRESERVE, - options::NO_PRESERVE, - ]) .help("Same as -dR --preserve=all") .action(ArgAction::SetTrue), ) @@ -658,44 +714,56 @@ pub fn uu_app() -> Command { Arg::new(options::SPARSE) .long(options::SPARSE) .value_name("WHEN") - .value_parser(["never", "auto", "always"]) + .value_parser(ShortcutValueParser::new(["never", "auto", "always"])) .help("control creation of sparse files. See below"), ) - // TODO: implement the following args .arg( - Arg::new(options::COPY_CONTENTS) - .long(options::COPY_CONTENTS) - .overrides_with(options::ATTRIBUTES_ONLY) - .help("NotImplemented: copy contents of special files when recursive") + Arg::new(options::SELINUX) + .short('Z') + .help("set SELinux security context of destination file to default type") .action(ArgAction::SetTrue), ) .arg( Arg::new(options::CONTEXT) .long(options::CONTEXT) .value_name("CTX") + .value_parser(value_parser!(String)) .help( - "NotImplemented: set SELinux security context of destination file to \ - default type", - ), + "like -Z, or if CTX is specified then set the SELinux or SMACK security \ + context to CTX", + ) + .num_args(0..=1) + .require_equals(true) + .default_missing_value(""), ) - // END TODO .arg( // The 'g' short flag is modeled after advcpmv // See this repo: https://github.com/jarun/advcpmv Arg::new(options::PROGRESS_BAR) .long(options::PROGRESS_BAR) .short('g') - .action(clap::ArgAction::SetTrue) + .action(ArgAction::SetTrue) .help( "Display a progress bar. \n\ Note: this feature is not supported by GNU coreutils.", ), ) + // TODO: implement the following args + .arg( + Arg::new(options::COPY_CONTENTS) + .long(options::COPY_CONTENTS) + .overrides_with(options::ATTRIBUTES_ONLY) + .help("NotImplemented: copy contents of special files when recursive") + .action(ArgAction::SetTrue), + ) + // END TODO .arg( Arg::new(options::PATHS) .action(ArgAction::Append) + .num_args(1..) + .required(true) .value_hint(clap::ValueHint::AnyPath) - .value_parser(ValueParser::path_buf()), + .value_parser(ValueParser::os_string()), ) } @@ -717,7 +785,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } else if let Ok(mut matches) = matches { let options = Options::from_matches(&matches)?; - if options.overwrite == OverwriteMode::NoClobber && options.backup != BackupMode::NoBackup { + if options.overwrite == OverwriteMode::NoClobber && options.backup != BackupMode::None { return Err(UUsageError::new( EXIT_ERR, "options --backup and --no-clobber are mutually exclusive", @@ -725,8 +793,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } let paths: Vec = matches - .remove_many::(options::PATHS) - .map(|v| v.collect()) + .remove_many::(options::PATHS) + .map(|v| v.map(PathBuf::from).collect()) .unwrap_or_default(); let (sources, target) = parse_path_args(paths, &options)?; @@ -737,7 +805,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { // code should still be EXIT_ERR as does GNU cp Error::NotAllFilesCopied => {} // Else we caught a fatal bubbled-up error, log it to stderr - _ => show_error!("{}", error), + _ => show_error!("{error}"), }; set_exit_code(EXIT_ERR); } @@ -783,7 +851,11 @@ impl CopyMode { { Self::Update } else if matches.get_flag(options::ATTRIBUTES_ONLY) { - Self::AttrOnly + if matches.get_flag(options::REMOVE_DESTINATION) { + Self::Copy + } else { + Self::AttrOnly + } } else { Self::Copy } @@ -847,6 +919,27 @@ impl Attributes { } } + /// Set the field to Preserve::NO { explicit: true } if the corresponding field + /// in other is set to Preserve::Yes { .. }. + pub fn diff(self, other: &Self) -> Self { + fn update_preserve_field(current: Preserve, other: Preserve) -> Preserve { + if matches!(other, Preserve::Yes { .. }) { + Preserve::No { explicit: true } + } else { + current + } + } + Self { + #[cfg(unix)] + ownership: update_preserve_field(self.ownership, other.ownership), + mode: update_preserve_field(self.mode, other.mode), + timestamps: update_preserve_field(self.timestamps, other.timestamps), + context: update_preserve_field(self.context, other.context), + links: update_preserve_field(self.links, other.links), + xattr: update_preserve_field(self.xattr, other.xattr), + } + } + pub fn parse_iter(values: impl Iterator) -> Result where T: AsRef, @@ -896,7 +989,6 @@ impl Options { let not_implemented_opts = vec![ #[cfg(not(any(windows, unix)))] options::ONE_FILE_SYSTEM, - options::CONTEXT, #[cfg(windows)] options::FORCE, ]; @@ -918,6 +1010,16 @@ impl Options { }; let update_mode = update_control::determine_update_mode(matches); + if backup_mode != BackupMode::None + && matches + .get_one::(update_control::arguments::OPT_UPDATE) + .is_some_and(|v| v == "none" || v == "none-fail") + { + return Err(Error::InvalidArgument( + "--backup is mutually exclusive with -n or --update=none-fail".to_string(), + )); + } + let backup_suffix = backup_control::determine_backup_suffix(matches); let overwrite = OverwriteMode::from_matches(matches); @@ -933,39 +1035,89 @@ impl Options { return Err(Error::NotADirectory(dir.clone())); } }; - - // Parse attributes to preserve - let mut attributes = - if let Some(attribute_strs) = matches.get_many::(options::PRESERVE) { - if attribute_strs.len() == 0 { - Attributes::DEFAULT - } else { - Attributes::parse_iter(attribute_strs)? + // cp follows POSIX conventions for overriding options such as "-a", + // "-d", "--preserve", and "--no-preserve". We can use clap's + // override-all behavior to achieve this, but there's a challenge: when + // clap overrides an argument, it removes all traces of it from the + // match. This poses a problem because flags like "-a" expand to "-dR + // --preserve=all", and we only want to override the "--preserve=all" + // part. Additionally, we need to handle multiple occurrences of the + // same flags. To address this, we create an overriding order from the + // matches here. + let mut overriding_order: Vec<(usize, &str, Vec<&String>)> = vec![]; + // We iterate through each overriding option, adding each occurrence of + // the option along with its value and index as a tuple, and push it to + // `overriding_order`. + for option in [ + options::PRESERVE, + options::NO_PRESERVE, + options::ARCHIVE, + options::PRESERVE_DEFAULT_ATTRIBUTES, + options::NO_DEREFERENCE_PRESERVE_LINKS, + ] { + if let (Ok(Some(val)), Some(index)) = ( + matches.try_get_one::(option), + // even though it says in the doc that `index_of` would give us + // the first index of the argument, when it comes to flag it + // gives us the last index where the flag appeared (probably + // because it overrides itself). Since it is a flag and it would + // have same value across the occurrences we just need the last + // index. + matches.index_of(option), + ) { + if *val { + overriding_order.push((index, option, vec![])); } - } else if matches.get_flag(options::ARCHIVE) { - // --archive is used. Same as --preserve=all - Attributes::ALL - } else if matches.get_flag(options::NO_DEREFERENCE_PRESERVE_LINKS) { - Attributes::LINKS - } else if matches.get_flag(options::PRESERVE_DEFAULT_ATTRIBUTES) { - Attributes::DEFAULT - } else { - Attributes::NONE - }; + } else if let (Some(occurrences), Some(mut indices)) = ( + matches.get_occurrences::(option), + matches.indices_of(option), + ) { + occurrences.for_each(|val| { + if let Some(index) = indices.next() { + let val = val.collect::>(); + // As mentioned in the documentation of the indices_of + // function, it provides the indices of the individual + // values. Therefore, to get the index of the first + // value of the next occurrence in the next iteration, + // we need to advance the indices iterator by the length + // of the current occurrence's values. + for _ in 1..val.len() { + indices.next(); + } + overriding_order.push((index, option, val)); + } + }); + } + } + overriding_order.sort_by(|a, b| a.0.cmp(&b.0)); - // handling no-preserve options and adjusting the attributes - if let Some(attribute_strs) = matches.get_many::(options::NO_PRESERVE) { - if attribute_strs.len() > 0 { - let no_preserve_attributes = Attributes::parse_iter(attribute_strs)?; - if matches!(no_preserve_attributes.links, Preserve::Yes { .. }) { - attributes.links = Preserve::No { explicit: true }; - } else if matches!(no_preserve_attributes.mode, Preserve::Yes { .. }) { - attributes.mode = Preserve::No { explicit: true }; + let mut attributes = Attributes::NONE; + + // Iterate through the `overriding_order` and adjust the attributes accordingly. + for (_, option, val) in overriding_order { + match option { + options::ARCHIVE => { + attributes = Attributes::ALL; + } + options::PRESERVE_DEFAULT_ATTRIBUTES => { + attributes = attributes.union(&Attributes::DEFAULT); + } + options::NO_DEREFERENCE_PRESERVE_LINKS => { + attributes = attributes.union(&Attributes::LINKS); + } + options::PRESERVE => { + attributes = attributes.union(&Attributes::parse_iter(val.into_iter())?); + } + options::NO_PRESERVE => { + if !val.is_empty() { + attributes = attributes.diff(&Attributes::parse_iter(val.into_iter())?); + } } + _ => (), } } - #[cfg(not(feature = "feat_selinux"))] + #[cfg(not(feature = "selinux"))] if let Preserve::Yes { required } = attributes.context { let selinux_disabled_error = Error::Error("SELinux was not enabled during the compile time!".to_string()); @@ -976,6 +1128,15 @@ impl Options { } } + // Extract the SELinux related flags and options + let set_selinux_context = matches.get_flag(options::SELINUX); + + let context = if matches.contains_id(options::CONTEXT) { + matches.get_one::(options::CONTEXT).cloned() + } else { + None + }; + let options = Self { attributes_only: matches.get_flag(options::ATTRIBUTES_ONLY), copy_contents: matches.get_flag(options::COPY_CONTENTS), @@ -985,7 +1146,9 @@ impl Options { dereference: !(matches.get_flag(options::NO_DEREFERENCE) || matches.get_flag(options::NO_DEREFERENCE_PRESERVE_LINKS) || matches.get_flag(options::ARCHIVE) - || recursive) + // cp normally follows the link only when not copying recursively or when + // --link (-l) is used + || (recursive && CopyMode::from_matches(matches)!= CopyMode::Link )) || matches.get_flag(options::DEREFERENCE), one_file_system: matches.get_flag(options::ONE_FILE_SYSTEM), parents: matches.get_flag(options::PARENTS), @@ -1007,18 +1170,7 @@ impl Options { } } } else { - #[cfg(any(target_os = "linux", target_os = "android", target_os = "macos"))] - { - ReflinkMode::Auto - } - #[cfg(not(any( - target_os = "linux", - target_os = "android", - target_os = "macos" - )))] - { - ReflinkMode::Never - } + ReflinkMode::default() } }, sparse_mode: { @@ -1045,6 +1197,8 @@ impl Options { recursive, target_dir, progress_bar: matches.get_flag(options::PROGRESS_BAR), + set_selinux_context: set_selinux_context || context.is_some(), + context, }; Ok(options) @@ -1064,10 +1218,13 @@ impl Options { #[cfg(unix)] fn preserve_mode(&self) -> (bool, bool) { match self.attributes.mode { - Preserve::No { explicit } => match explicit { - true => (false, true), - false => (false, false), - }, + Preserve::No { explicit } => { + if explicit { + (false, true) + } else { + (false, false) + } + } Preserve::Yes { .. } => (true, false), } } @@ -1125,6 +1282,9 @@ fn parse_path_args( }; if options.strip_trailing_slashes { + // clippy::assigning_clones added with Rust 1.78 + // Rust version = 1.76 on OpenBSD stable/7.5 + #[cfg_attr(not(target_os = "openbsd"), allow(clippy::assigning_clones))] for source in &mut paths { *source = source.components().as_path().to_owned(); } @@ -1141,12 +1301,12 @@ fn show_error_if_needed(error: &Error) { Error::NotAllFilesCopied => { // Need to return an error code } - Error::Skipped => { + Error::Skipped(_) => { // touch a b && echo "n"|cp -i a b && echo $? // should return an error from GNU 9.2 } _ => { - show_error!("{}", error); + show_error!("{error}"); } } } @@ -1193,19 +1353,30 @@ pub fn copy(sources: &[PathBuf], target: &Path, options: &Options) -> CopyResult }; for source in sources { - if seen_sources.contains(source) { - // FIXME: compare sources by the actual file they point to, not their path. (e.g. dir/file == dir/../dir/file in most cases) - show_warning!("source file {} specified more than once", source.quote()); + let normalized_source = normalize_path(source); + if options.backup == BackupMode::None && seen_sources.contains(&normalized_source) { + let file_type = if source.symlink_metadata()?.file_type().is_dir() { + "directory" + } else { + "file" + }; + show_warning!( + "source {file_type} {} specified more than once", + source.quote() + ); } else { let dest = construct_dest_path(source, target, target_type, options) .unwrap_or_else(|_| target.to_path_buf()); - if fs::metadata(&dest).is_ok() && !fs::symlink_metadata(&dest)?.file_type().is_symlink() + if fs::metadata(&dest).is_ok() + && !fs::symlink_metadata(&dest)?.file_type().is_symlink() + // if both `source` and `dest` are symlinks, it should be considered as an overwrite. + || fs::metadata(source).is_ok() + && fs::symlink_metadata(source)?.file_type().is_symlink() + || matches!(options.copy_mode, CopyMode::SymLink) { // There is already a file and it isn't a symlink (managed in a different place) - if copied_destinations.contains(&dest) - && options.backup != BackupMode::NumberedBackup - { + if copied_destinations.contains(&dest) && options.backup != BackupMode::Numbered { // If the target file was already created in this cp call, do not overwrite return Err(Error::Error(format!( "will not overwrite just-created '{}' with '{}'", @@ -1216,20 +1387,24 @@ pub fn copy(sources: &[PathBuf], target: &Path, options: &Options) -> CopyResult } if let Err(error) = copy_source( - &progress_bar, + progress_bar.as_ref(), source, target, target_type, options, &mut symlinked_files, + &copied_destinations, &mut copied_files, ) { show_error_if_needed(&error); - non_fatal_errors = true; + if !matches!(error, Error::Skipped(false)) { + non_fatal_errors = true; + } + } else { + copied_destinations.insert(dest.clone()); } - copied_destinations.insert(dest.clone()); } - seen_sources.insert(source); + seen_sources.insert(normalized_source); } if let Some(pb) = progress_bar { @@ -1264,7 +1439,11 @@ fn construct_dest_path( Ok(match target_type { TargetType::Directory => { let root = if options.parents { - Path::new("") + if source_path.has_root() && cfg!(unix) { + Path::new("/") + } else { + Path::new("") + } } else { source_path.parent().unwrap_or(source_path) }; @@ -1273,14 +1452,15 @@ fn construct_dest_path( TargetType::File => target.to_path_buf(), }) } - +#[allow(clippy::too_many_arguments)] fn copy_source( - progress_bar: &Option, + progress_bar: Option<&ProgressBar>, source: &Path, target: &Path, target_type: TargetType, options: &Options, symlinked_files: &mut HashSet, + copied_destinations: &HashSet, copied_files: &mut HashMap, ) -> CopyResult<()> { let source_path = Path::new(&source); @@ -1292,6 +1472,7 @@ fn copy_source( target, options, symlinked_files, + copied_destinations, copied_files, true, ) @@ -1304,30 +1485,99 @@ fn copy_source( dest.as_path(), options, symlinked_files, + copied_destinations, copied_files, true, ); if options.parents { for (x, y) in aligned_ancestors(source, dest.as_path()) { - copy_attributes(x, y, &options.attributes)?; + if let Ok(src) = canonicalize(x, MissingHandling::Normal, ResolveMode::Physical) { + copy_attributes(&src, y, &options.attributes)?; + } } } res } } +/// If `path` does not have `S_IWUSR` set, returns a tuple of the file's +/// mode in octal (index 0) and human-readable (index 1) formats. +/// +/// If the destination of a copy operation is a file that is not writeable to +/// the owner (bit `S_IWUSR`), extra information needs to be added to the +/// interactive mode prompt: the mode (permissions) of the file in octal and +/// human-readable format. +// TODO +// The destination metadata can be read multiple times in the course of a single execution of `cp`. +// This fix adds yet another metadata read. +// Should this metadata be read once and then reused throughout the execution? +// https://github.com/uutils/coreutils/issues/6658 +fn file_mode_for_interactive_overwrite( + #[cfg_attr(not(unix), allow(unused_variables))] path: &Path, +) -> Option<(String, String)> { + // Retain outer braces to ensure only one branch is included + { + #[cfg(unix)] + { + use libc::{S_IWUSR, mode_t}; + use std::os::unix::prelude::MetadataExt; + + match path.metadata() { + Ok(me) => { + // Cast is necessary on some platforms + let mode: mode_t = me.mode() as mode_t; + + // It looks like this extra information is added to the prompt iff the file's user write bit is 0 + // write permission, owner + if uucore::has!(mode, S_IWUSR) { + None + } else { + // Discard leading digits + let mode_without_leading_digits = mode & 0o7777; + + Some(( + format!("{mode_without_leading_digits:04o}"), + uucore::fs::display_permissions_unix(mode, false), + )) + } + } + // TODO: How should failure to read the metadata be handled? Ignoring for now. + Err(_) => None, + } + } + + #[cfg(not(unix))] + { + None + } + } +} + impl OverwriteMode { - fn verify(&self, path: &Path) -> CopyResult<()> { + fn verify(&self, path: &Path, debug: bool) -> CopyResult<()> { match *self { Self::NoClobber => { - eprintln!("{}: not replacing {}", util_name(), path.quote()); - Err(Error::NotAllFilesCopied) + if debug { + println!("skipped {}", path.quote()); + } + Err(Error::Skipped(false)) } Self::Interactive(_) => { - if prompt_yes!("overwrite {}?", path.quote()) { + let prompt_yes_result = if let Some((octal, human_readable)) = + file_mode_for_interactive_overwrite(path) + { + prompt_yes!( + "replace {}, overriding mode {octal} ({human_readable})?", + path.quote() + ) + } else { + prompt_yes!("overwrite {}?", path.quote()) + }; + + if prompt_yes_result { Ok(()) } else { - Err(Error::Skipped) + Err(Error::Skipped(true)) } } Self::Clobber(_) => Ok(()), @@ -1353,6 +1603,42 @@ fn handle_preserve CopyResult<()>>(p: &Preserve, f: F) -> CopyResult< Ok(()) } +/// Copies extended attributes (xattrs) from `source` to `dest`, ensuring that `dest` is temporarily +/// user-writable if needed and restoring its original permissions afterward. This avoids “Operation +/// not permitted†errors on read-only files. Returns an error if permission or metadata operations fail, +/// or if xattr copying fails. +#[cfg(all(unix, not(target_os = "android")))] +fn copy_extended_attrs(source: &Path, dest: &Path) -> CopyResult<()> { + let metadata = fs::symlink_metadata(dest)?; + + // Check if the destination file is currently read-only for the user. + let mut perms = metadata.permissions(); + let was_readonly = perms.readonly(); + + // Temporarily grant user write if it was read-only. + if was_readonly { + #[allow(clippy::permissions_set_readonly_false)] + perms.set_readonly(false); + fs::set_permissions(dest, perms)?; + } + + // Perform the xattr copy and capture any potential error, + // so we can restore permissions before returning. + let copy_xattrs_result = copy_xattrs(source, dest); + + // Restore read-only if we changed it. + if was_readonly { + let mut revert_perms = fs::symlink_metadata(dest)?.permissions(); + revert_perms.set_readonly(true); + fs::set_permissions(dest, revert_perms)?; + } + + // If copying xattrs failed, propagate that error now. + copy_xattrs_result?; + + Ok(()) +} + /// Copy the specified attributes from one path to another. pub(crate) fn copy_attributes( source: &Path, @@ -1366,14 +1652,14 @@ pub(crate) fn copy_attributes( #[cfg(unix)] handle_preserve(&attributes.ownership, || -> CopyResult<()> { use std::os::unix::prelude::MetadataExt; - use uucore::perms::wrap_chown; use uucore::perms::Verbosity; use uucore::perms::VerbosityLevel; + use uucore::perms::wrap_chown; let dest_uid = source_metadata.uid(); let dest_gid = source_metadata.gid(); - - wrap_chown( + // gnu compatibility: cp doesn't report an error if it fails to set the ownership. + let _ = wrap_chown( dest, &dest.symlink_metadata().context(context)?, Some(dest_uid), @@ -1381,11 +1667,9 @@ pub(crate) fn copy_attributes( false, Verbosity { groups_only: false, - level: VerbosityLevel::Normal, + level: VerbosityLevel::Silent, }, - ) - .map_err(Error::Error)?; - + ); Ok(()) })?; @@ -1419,37 +1703,31 @@ pub(crate) fn copy_attributes( Ok(()) })?; - #[cfg(feature = "feat_selinux")] + #[cfg(feature = "selinux")] handle_preserve(&attributes.context, || -> CopyResult<()> { - let context = selinux::SecurityContext::of_path(source, false, false).map_err(|e| { - format!( - "failed to get security context of {}: {}", - source.display(), - e - ) - })?; - if let Some(context) = context { - context.set_for_path(dest, false, false).map_err(|e| { - format!( - "failed to set security context for {}: {}", - dest.display(), - e - ) - })?; + // Get the source context and apply it to the destination + if let Ok(context) = selinux::SecurityContext::of_path(source, false, false) { + if let Some(context) = context { + if let Err(e) = context.set_for_path(dest, false, false) { + return Err(Error::Error(format!( + "failed to set the security context of {}: {e}", + dest.display() + ))); + } + } + } else { + return Err(Error::Error(format!( + "failed to get security context of {}", + source.display() + ))); } - Ok(()) })?; handle_preserve(&attributes.xattr, || -> CopyResult<()> { #[cfg(all(unix, not(target_os = "android")))] { - let xattrs = xattr::list(source)?; - for attr in xattrs { - if let Some(attr_value) = xattr::get(source, attr.clone())? { - xattr::set(dest, attr, &attr_value[..])?; - } - } + copy_extended_attrs(source, dest)?; } #[cfg(not(all(unix, not(target_os = "android"))))] { @@ -1474,16 +1752,23 @@ pub(crate) fn copy_attributes( fn symlink_file( source: &Path, dest: &Path, - context: &str, symlinked_files: &mut HashSet, ) -> CopyResult<()> { #[cfg(not(windows))] { - std::os::unix::fs::symlink(source, dest).context(context)?; + std::os::unix::fs::symlink(source, dest).context(format!( + "cannot create symlink {} to {}", + get_filename(dest).unwrap_or("invalid file name").quote(), + get_filename(source).unwrap_or("invalid file name").quote() + ))?; } #[cfg(windows)] { - std::os::windows::fs::symlink_file(source, dest).context(context)?; + std::os::windows::fs::symlink_file(source, dest).context(format!( + "cannot create symlink {} to {}", + get_filename(dest).unwrap_or("invalid file name").quote(), + get_filename(source).unwrap_or("invalid file name").quote() + ))?; } if let Ok(file_info) = FileInformation::from_path(dest, false) { symlinked_files.insert(file_info); @@ -1495,10 +1780,11 @@ fn context_for(src: &Path, dest: &Path) -> String { format!("{} -> {}", src.quote(), dest.quote()) } -/// Implements a simple backup copy for the destination file. +/// Implements a simple backup copy for the destination file . +/// if is_dest_symlink flag is set to true dest will be renamed to backup_path /// TODO: for the backup, should this function be replaced by `copy_file(...)`? -fn backup_dest(dest: &Path, backup_path: &Path) -> CopyResult { - if dest.is_symlink() { +fn backup_dest(dest: &Path, backup_path: &Path, is_dest_symlink: bool) -> CopyResult { + if is_dest_symlink { fs::rename(dest, backup_path)?; } else { fs::copy(dest, backup_path)?; @@ -1519,11 +1805,44 @@ fn is_forbidden_to_copy_to_same_file( ) -> bool { // TODO To match the behavior of GNU cp, we also need to check // that the file is a regular file. + let source_is_symlink = source.is_symlink(); + let dest_is_symlink = dest.is_symlink(); + // only disable dereference if both source and dest is symlink and dereference flag is disabled let dereference_to_compare = - options.dereference(source_in_command_line) || !source.is_symlink(); - paths_refer_to_same_file(source, dest, dereference_to_compare) - && !(options.force() && options.backup != BackupMode::NoBackup) - && !(dest.is_symlink() && options.backup != BackupMode::NoBackup) + options.dereference(source_in_command_line) || (!source_is_symlink || !dest_is_symlink); + if !paths_refer_to_same_file(source, dest, dereference_to_compare) { + return false; + } + if options.backup != BackupMode::None { + if options.force() && !source_is_symlink { + return false; + } + if source_is_symlink && !options.dereference { + return false; + } + if dest_is_symlink { + return false; + } + if !dest_is_symlink && !source_is_symlink && dest != source { + return false; + } + } + if options.copy_mode == CopyMode::Link { + return false; + } + if options.copy_mode == CopyMode::SymLink && dest_is_symlink { + return false; + } + // If source and dest are both the same symlink but with different names, then allow the copy. + // This can occur, for example, if source and dest are both hardlinks to the same symlink. + if dest_is_symlink + && source_is_symlink + && source.file_name() != dest.file_name() + && !options.dereference + { + return false; + } + true } /// Back up, remove, or leave intact the destination file, depending on the options. @@ -1532,6 +1851,7 @@ fn handle_existing_dest( dest: &Path, options: &Options, source_in_command_line: bool, + copied_files: &HashMap, ) -> CopyResult<()> { // Disallow copying a file to itself, unless `--force` and // `--backup` are both specified. @@ -1539,8 +1859,18 @@ fn handle_existing_dest( return Err(format!("{} and {} are the same file", source.quote(), dest.quote()).into()); } - options.overwrite.verify(dest)?; + if options.update == UpdateMode::None { + if options.debug { + println!("skipped {}", dest.quote()); + } + return Err(Error::Skipped(false)); + } + + if options.update != UpdateMode::IfOlder { + options.overwrite.verify(dest, options.debug)?; + } + let mut is_dest_removed = false; let backup_path = backup_control::get_backup_path(options.backup, dest, &options.backup_suffix); if let Some(backup_path) = backup_path { if paths_refer_to_same_file(source, &backup_path, true) { @@ -1551,40 +1881,80 @@ fn handle_existing_dest( ) .into()); } else { - backup_dest(dest, &backup_path)?; + is_dest_removed = dest.is_symlink(); + backup_dest(dest, &backup_path, is_dest_removed)?; } } - match options.overwrite { - // FIXME: print that the file was removed if --verbose is enabled - OverwriteMode::Clobber(ClobberMode::Force) => { - if is_symlink_loop(dest) || fs::metadata(dest)?.permissions().readonly() { - fs::remove_file(dest)?; - } - } - OverwriteMode::Clobber(ClobberMode::RemoveDestination) => { - fs::remove_file(dest)?; - } - OverwriteMode::Clobber(ClobberMode::Standard) => { - // Consider the following files: - // - // * `src/f` - a regular file - // * `src/link` - a hard link to `src/f` - // * `dest/src/f` - a different regular file - // - // In this scenario, if we do `cp -a src/ dest/`, it is - // possible that the order of traversal causes `src/link` - // to get copied first (to `dest/src/link`). In that case, - // in order to make sure `dest/src/link` is a hard link to - // `dest/src/f` and `dest/src/f` has the contents of - // `src/f`, we delete the existing file to allow the hard - // linking. - if options.preserve_hard_links() { - fs::remove_file(dest)?; + if !is_dest_removed { + delete_dest_if_needed_and_allowed( + source, + dest, + options, + source_in_command_line, + copied_files, + )?; + } + + Ok(()) +} + +/// Checks if: +/// * `dest` needs to be deleted before the copy operation can proceed +/// * the provided options allow this deletion +/// +/// If so, deletes `dest`. +fn delete_dest_if_needed_and_allowed( + source: &Path, + dest: &Path, + options: &Options, + source_in_command_line: bool, + copied_files: &HashMap, +) -> CopyResult<()> { + let delete_dest = match options.overwrite { + OverwriteMode::Clobber(cl) | OverwriteMode::Interactive(cl) => { + match cl { + // FIXME: print that the file was removed if --verbose is enabled + ClobberMode::Force => { + // TODO + // Using `readonly` here to check if `dest` needs to be deleted is not correct: + // "On Unix-based platforms this checks if any of the owner, group or others write permission bits are set. It does not check if the current user is in the file's assigned group. It also does not check ACLs. Therefore the return value of this function cannot be relied upon to predict whether attempts to read or write the file will actually succeed." + // This results in some copy operations failing, because this necessary deletion is being skipped. + is_symlink_loop(dest) || fs::metadata(dest)?.permissions().readonly() + } + ClobberMode::RemoveDestination => true, + ClobberMode::Standard => { + // Consider the following files: + // + // * `src/f` - a regular file + // * `src/link` - a hard link to `src/f` + // * `dest/src/f` - a different regular file + // + // In this scenario, if we do `cp -a src/ dest/`, it is + // possible that the order of traversal causes `src/link` + // to get copied first (to `dest/src/link`). In that case, + // in order to make sure `dest/src/link` is a hard link to + // `dest/src/f` and `dest/src/f` has the contents of + // `src/f`, we delete the existing file to allow the hard + // linking. + options.preserve_hard_links() && + // only try to remove dest file only if the current source + // is hardlink to a file that is already copied + copied_files.contains_key( + &FileInformation::from_path( + source, + options.dereference(source_in_command_line) + ).context(format!("cannot stat {}", source.quote()))? + ) + } } } - _ => (), + OverwriteMode::NoClobber => false, }; + if delete_dest { + fs::remove_file(dest)?; + } + Ok(()) } @@ -1624,7 +1994,7 @@ fn aligned_ancestors<'a>(source: &'a Path, dest: &'a Path) -> Vec<(&'a Path, &'a // Get the matching number of elements from the ancestors of the // destination path (for example, get "d/a" and "d/a/b"). let k = source_ancestors.len(); - let dest_ancestors = &dest_ancestors[1..1 + k]; + let dest_ancestors = &dest_ancestors[1..=k]; // Now we have two slices of the same length, so we zip them. let mut result = vec![]; @@ -1638,177 +2008,61 @@ fn aligned_ancestors<'a>(source: &'a Path, dest: &'a Path) -> Vec<(&'a Path, &'a result } -/// Copy the a file from `source` to `dest`. `source` will be dereferenced if -/// `options.dereference` is set to true. `dest` will be dereferenced only if -/// the source was not a symlink. -/// -/// Behavior when copying to existing files is contingent on the -/// `options.overwrite` mode. If a file is skipped, the return type -/// should be `Error:Skipped` -/// -/// The original permissions of `source` will be copied to `dest` -/// after a successful copy. -#[allow(clippy::cognitive_complexity)] -fn copy_file( - progress_bar: &Option, +fn print_verbose_output( + parents: bool, + progress_bar: Option<&ProgressBar>, source: &Path, dest: &Path, - options: &Options, - symlinked_files: &mut HashSet, - copied_files: &mut HashMap, - source_in_command_line: bool, -) -> CopyResult<()> { - if (options.update == UpdateMode::ReplaceIfOlder || options.update == UpdateMode::ReplaceNone) - && options.overwrite == OverwriteMode::Interactive(ClobberMode::Standard) - { - // `cp -i --update old new` when `new` exists doesn't copy anything - // and exit with 0 - return Ok(()); - } - - // Fail if dest is a dangling symlink or a symlink this program created previously - if dest.is_symlink() { - if FileInformation::from_path(dest, false) - .map(|info| symlinked_files.contains(&info)) - .unwrap_or(false) - { - return Err(Error::Error(format!( - "will not copy '{}' through just-created symlink '{}'", - source.display(), - dest.display() - ))); - } - let copy_contents = options.dereference(source_in_command_line) || !source.is_symlink(); - if copy_contents - && !dest.exists() - && !matches!( - options.overwrite, - OverwriteMode::Clobber(ClobberMode::RemoveDestination) - ) - && !is_symlink_loop(dest) - && std::env::var_os("POSIXLY_CORRECT").is_none() - { - return Err(Error::Error(format!( - "not writing through dangling symlink '{}'", - dest.display() - ))); - } - if paths_refer_to_same_file(source, dest, true) - && matches!( - options.overwrite, - OverwriteMode::Clobber(ClobberMode::RemoveDestination) - ) - { - fs::remove_file(dest)?; - } - } - - if are_hardlinks_to_same_file(source, dest) - && matches!( - options.overwrite, - OverwriteMode::Clobber(ClobberMode::RemoveDestination) - ) - { - fs::remove_file(dest)?; - } - - if file_or_link_exists(dest) { - if are_hardlinks_to_same_file(source, dest) - && !options.force() - && options.backup == BackupMode::NoBackup - && source != dest - || (source == dest && options.copy_mode == CopyMode::Link) - { - return Ok(()); - } - handle_existing_dest(source, dest, options, source_in_command_line)?; - } - - if options.preserve_hard_links() { - // if we encounter a matching device/inode pair in the source tree - // we can arrange to create a hard link between the corresponding names - // in the destination tree. - if let Some(new_source) = copied_files.get( - &FileInformation::from_path(source, options.dereference(source_in_command_line)) - .context(format!("cannot stat {}", source.quote()))?, - ) { - std::fs::hard_link(new_source, dest)?; - return Ok(()); - }; +) { + if let Some(pb) = progress_bar { + // Suspend (hide) the progress bar so the println won't overlap with the progress bar. + pb.suspend(|| { + print_paths(parents, source, dest); + }); + } else { + print_paths(parents, source, dest); } +} - if options.verbose { - if let Some(pb) = progress_bar { - // Suspend (hide) the progress bar so the println won't overlap with the progress bar. - pb.suspend(|| { - if options.parents { - // For example, if copying file `a/b/c` and its parents - // to directory `d/`, then print - // - // a -> d/a - // a/b -> d/a/b - // - for (x, y) in aligned_ancestors(source, dest) { - println!("{} -> {}", x.display(), y.display()); - } - } - - println!("{}", context_for(source, dest)); - }); - } else { - if options.parents { - // For example, if copying file `a/b/c` and its parents - // to directory `d/`, then print - // - // a -> d/a - // a/b -> d/a/b - // - for (x, y) in aligned_ancestors(source, dest) { - println!("{} -> {}", x.display(), y.display()); - } - } - - println!("{}", context_for(source, dest)); +fn print_paths(parents: bool, source: &Path, dest: &Path) { + if parents { + // For example, if copying file `a/b/c` and its parents + // to directory `d/`, then print + // + // a -> d/a + // a/b -> d/a/b + // + for (x, y) in aligned_ancestors(source, dest) { + println!("{} -> {}", x.display(), y.display()); } } - // Calculate the context upfront before canonicalizing the path - let context = context_for(source, dest); - let context = context.as_str(); - - let source_metadata = { - let result = if options.dereference(source_in_command_line) { - fs::metadata(source) - } else { - fs::symlink_metadata(source) - }; - result.context(context)? - }; - let source_file_type = source_metadata.file_type(); - let source_is_symlink = source_file_type.is_symlink(); - - #[cfg(unix)] - let source_is_fifo = source_file_type.is_fifo(); - #[cfg(not(unix))] - let source_is_fifo = false; - - let dest_permissions = if dest.exists() { - dest.symlink_metadata().context(context)?.permissions() - } else { - #[allow(unused_mut)] - let mut permissions = source_metadata.permissions(); - #[cfg(unix)] - { - let mut mode = handle_no_preserve_mode(options, permissions.mode()); - - // apply umask - use uucore::mode::get_umask; - mode &= !get_umask(); + println!("{}", context_for(source, dest)); +} - permissions.set_mode(mode); - } - permissions - }; +/// Handles the copy mode for a file copy operation. +/// +/// This function determines how to copy a file based on the provided options. +/// It supports different copy modes, including hard linking, copying, symbolic linking, updating, and attribute-only copying. +/// It also handles file backups, overwriting, and dereferencing based on the provided options. +/// +/// # Returns +/// +/// * `Ok(())` - The file was copied successfully. +/// * `Err(CopyError)` - An error occurred while copying the file. +#[allow(clippy::too_many_arguments)] +fn handle_copy_mode( + source: &Path, + dest: &Path, + options: &Options, + context: &str, + source_metadata: &Metadata, + symlinked_files: &mut HashSet, + source_in_command_line: bool, + source_is_fifo: bool, + #[cfg(unix)] source_is_stream: bool, +) -> CopyResult { + let source_is_symlink = source_metadata.is_symlink(); match options.copy_mode { CopyMode::Link => { @@ -1816,7 +2070,7 @@ fn copy_file( let backup_path = backup_control::get_backup_path(options.backup, dest, &options.backup_suffix); if let Some(backup_path) = backup_path { - backup_dest(dest, &backup_path)?; + backup_dest(dest, &backup_path, dest.is_symlink())?; fs::remove_file(dest)?; } if options.overwrite == OverwriteMode::Clobber(ClobberMode::Force) { @@ -1830,7 +2084,11 @@ fn copy_file( } else { fs::hard_link(source, dest) } - .context(context)?; + .context(format!( + "cannot create hard link {} to {}", + get_filename(dest).unwrap_or("invalid file name").quote(), + get_filename(source).unwrap_or("invalid file name").quote() + ))?; } CopyMode::Copy => { copy_helper( @@ -1841,18 +2099,20 @@ fn copy_file( source_is_symlink, source_is_fifo, symlinked_files, + #[cfg(unix)] + source_is_stream, )?; } CopyMode::SymLink => { if dest.exists() && options.overwrite == OverwriteMode::Clobber(ClobberMode::Force) { fs::remove_file(dest)?; } - symlink_file(source, dest, context, symlinked_files)?; + symlink_file(source, dest, symlinked_files)?; } CopyMode::Update => { if dest.exists() { match options.update { - update_control::UpdateMode::ReplaceAll => { + UpdateMode::All => { copy_helper( source, dest, @@ -1861,23 +2121,30 @@ fn copy_file( source_is_symlink, source_is_fifo, symlinked_files, + #[cfg(unix)] + source_is_stream, )?; } - update_control::UpdateMode::ReplaceNone => { + UpdateMode::None => { if options.debug { println!("skipped {}", dest.quote()); } - return Ok(()); + return Ok(PerformedAction::Skipped); + } + UpdateMode::NoneFail => { + return Err(Error::Error(format!("not replacing '{}'", dest.display()))); } - update_control::UpdateMode::ReplaceIfOlder => { + UpdateMode::IfOlder => { let dest_metadata = fs::symlink_metadata(dest)?; let src_time = source_metadata.modified()?; let dest_time = dest_metadata.modified()?; if src_time <= dest_time { - return Ok(()); + return Ok(PerformedAction::Skipped); } else { + options.overwrite.verify(dest, options.debug)?; + copy_helper( source, dest, @@ -1886,6 +2153,8 @@ fn copy_file( source_is_symlink, source_is_fifo, symlinked_files, + #[cfg(unix)] + source_is_stream, )?; } } @@ -1899,6 +2168,8 @@ fn copy_file( source_is_symlink, source_is_fifo, symlinked_files, + #[cfg(unix)] + source_is_stream, )?; } } @@ -1912,8 +2183,251 @@ fn copy_file( } }; + Ok(PerformedAction::Copied) +} + +/// Calculates the permissions for the destination file in a copy operation. +/// +/// If the destination file already exists, its current permissions are returned. +/// If the destination file does not exist, the source file's permissions are used, +/// with the `no-preserve` option and the umask taken into account on Unix platforms. +/// # Returns +/// +/// * `Ok(Permissions)` - The calculated permissions for the destination file. +/// * `Err(CopyError)` - An error occurred while getting the metadata of the destination file. +/// +// Allow unused variables for Windows (on options) +#[allow(unused_variables)] +fn calculate_dest_permissions( + dest: &Path, + source_metadata: &Metadata, + options: &Options, + context: &str, +) -> CopyResult { + if dest.exists() { + Ok(dest.symlink_metadata().context(context)?.permissions()) + } else { + #[cfg(unix)] + { + let mut permissions = source_metadata.permissions(); + let mode = handle_no_preserve_mode(options, permissions.mode()); + + // Apply umask + use uucore::mode::get_umask; + let mode = mode & !get_umask(); + permissions.set_mode(mode); + Ok(permissions) + } + #[cfg(not(unix))] + { + let permissions = source_metadata.permissions(); + Ok(permissions) + } + } +} + +/// Copy the a file from `source` to `dest`. `source` will be dereferenced if +/// `options.dereference` is set to true. `dest` will be dereferenced only if +/// the source was not a symlink. +/// +/// Behavior when copying to existing files is contingent on the +/// `options.overwrite` mode. If a file is skipped, the return type +/// should be `Error:Skipped` +/// +/// The original permissions of `source` will be copied to `dest` +/// after a successful copy. +#[allow(clippy::cognitive_complexity, clippy::too_many_arguments)] +fn copy_file( + progress_bar: Option<&ProgressBar>, + source: &Path, + dest: &Path, + options: &Options, + symlinked_files: &mut HashSet, + copied_destinations: &HashSet, + copied_files: &mut HashMap, + source_in_command_line: bool, +) -> CopyResult<()> { + let source_is_symlink = source.is_symlink(); + let dest_is_symlink = dest.is_symlink(); + // Fail if dest is a dangling symlink or a symlink this program created previously + if dest_is_symlink { + if FileInformation::from_path(dest, false) + .map(|info| symlinked_files.contains(&info)) + .unwrap_or(false) + { + return Err(Error::Error(format!( + "will not copy '{}' through just-created symlink '{}'", + source.display(), + dest.display() + ))); + } + // Fail if cp tries to copy two sources of the same name into a single symlink + // Example: "cp file1 dir1/file1 tmp" where "tmp" is a directory containing a symlink "file1" pointing to a file named "foo". + // foo will contain the contents of "file1" and "dir1/file1" will not be copied over to "tmp/file1" + if copied_destinations.contains(dest) { + return Err(Error::Error(format!( + "will not copy '{}' through just-created symlink '{}'", + source.display(), + dest.display() + ))); + } + + let copy_contents = options.dereference(source_in_command_line) || !source_is_symlink; + if copy_contents + && !dest.exists() + && !matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + ) + && !is_symlink_loop(dest) + && std::env::var_os("POSIXLY_CORRECT").is_none() + { + return Err(Error::Error(format!( + "not writing through dangling symlink '{}'", + dest.display() + ))); + } + if paths_refer_to_same_file(source, dest, true) + && matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + ) + && options.backup == BackupMode::None + { + fs::remove_file(dest)?; + } + } + + if are_hardlinks_to_same_file(source, dest) + && source != dest + && matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + ) + { + fs::remove_file(dest)?; + } + + if file_or_link_exists(dest) + && (!options.attributes_only + || matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + )) + { + if paths_refer_to_same_file(source, dest, true) && options.copy_mode == CopyMode::Link { + if source_is_symlink { + if !dest_is_symlink { + return Ok(()); + } + if !options.dereference { + return Ok(()); + } + } else if options.backup != BackupMode::None && !dest_is_symlink { + if source == dest { + if !options.force() { + return Ok(()); + } + } else { + return Ok(()); + } + } + } + handle_existing_dest(source, dest, options, source_in_command_line, copied_files)?; + if are_hardlinks_to_same_file(source, dest) { + if options.copy_mode == CopyMode::Copy { + return Ok(()); + } + if options.copy_mode == CopyMode::Link && (!source_is_symlink || !dest_is_symlink) { + return Ok(()); + } + } + } + + if options.attributes_only + && source_is_symlink + && !matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + ) + { + return Err(format!( + "cannot change attribute {}: Source file is a non regular file", + dest.quote() + ) + .into()); + } + + if options.preserve_hard_links() { + // if we encounter a matching device/inode pair in the source tree + // we can arrange to create a hard link between the corresponding names + // in the destination tree. + if let Some(new_source) = copied_files.get( + &FileInformation::from_path(source, options.dereference(source_in_command_line)) + .context(format!("cannot stat {}", source.quote()))?, + ) { + fs::hard_link(new_source, dest)?; + + if options.verbose { + print_verbose_output(options.parents, progress_bar, source, dest); + } + + return Ok(()); + }; + } + + // Calculate the context upfront before canonicalizing the path + let context = context_for(source, dest); + let context = context.as_str(); + + let source_metadata = { + let result = if options.dereference(source_in_command_line) { + fs::metadata(source) + } else { + fs::symlink_metadata(source) + }; + // this is just for gnu tests compatibility + result.map_err(|err| { + if err.to_string().contains("No such file or directory") { + return format!("cannot stat {}: No such file or directory", source.quote()); + } + err.to_string() + })? + }; + + let dest_permissions = calculate_dest_permissions(dest, &source_metadata, options, context)?; + + #[cfg(unix)] + let source_is_fifo = source_metadata.file_type().is_fifo(); + #[cfg(not(unix))] + let source_is_fifo = false; + + #[cfg(unix)] + let source_is_stream = source_is_fifo + || source_metadata.file_type().is_char_device() + || source_metadata.file_type().is_block_device(); + #[cfg(not(unix))] + let source_is_stream = false; + + let performed_action = handle_copy_mode( + source, + dest, + options, + context, + &source_metadata, + symlinked_files, + source_in_command_line, + source_is_fifo, + #[cfg(unix)] + source_is_stream, + )?; + + if options.verbose && performed_action != PerformedAction::Skipped { + print_verbose_output(options.parents, progress_bar, source, dest); + } + // TODO: implement something similar to gnu's lchown - if !dest.is_symlink() { + if !dest_is_symlink { // Here, to match GNU semantics, we quietly ignore an error // if a user does not have the correct ownership to modify // the permissions of a file. @@ -1923,7 +2437,30 @@ fn copy_file( fs::set_permissions(dest, dest_permissions).ok(); } - copy_attributes(source, dest, &options.attributes)?; + if options.dereference(source_in_command_line) { + if let Ok(src) = canonicalize(source, MissingHandling::Normal, ResolveMode::Physical) { + if src.exists() { + copy_attributes(&src, dest, &options.attributes)?; + } + } + } else if source_is_stream && source.exists() { + // Some stream files may not exist after we have copied it, + // like anonymous pipes. Thus, we can't really copy its + // attributes. However, this is already handled in the stream + // copy function (see `copy_stream` under platform/linux.rs). + } else { + copy_attributes(source, dest, &options.attributes)?; + } + + #[cfg(feature = "selinux")] + if options.set_selinux_context && uucore::selinux::is_selinux_enabled() { + // Set the given selinux permissions on the copied file. + if let Err(e) = + uucore::selinux::set_selinux_security_context(dest, options.context.as_ref()) + { + return Err(Error::Error(format!("SELinux error: {}", e))); + } + } copied_files.insert( FileInformation::from_path(source, options.dereference(source_in_command_line))?, @@ -1948,23 +2485,22 @@ fn handle_no_preserve_mode(options: &Options, org_mode: u32) -> u32 { #[cfg(not(any( target_os = "android", target_os = "macos", - target_os = "macos-12", target_os = "freebsd", target_os = "redox", )))] { const MODE_RW_UGO: u32 = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; const S_IRWXUGO: u32 = S_IRWXU | S_IRWXG | S_IRWXO; - match is_explicit_no_preserve_mode { - true => return MODE_RW_UGO, - false => return org_mode & S_IRWXUGO, + return if is_explicit_no_preserve_mode { + MODE_RW_UGO + } else { + org_mode & S_IRWXUGO }; } #[cfg(any( target_os = "android", target_os = "macos", - target_os = "macos-12", target_os = "freebsd", target_os = "redox", ))] @@ -1972,9 +2508,10 @@ fn handle_no_preserve_mode(options: &Options, org_mode: u32) -> u32 { const MODE_RW_UGO: u32 = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH) as u32; const S_IRWXUGO: u32 = (S_IRWXU | S_IRWXG | S_IRWXO) as u32; - match is_explicit_no_preserve_mode { - true => return MODE_RW_UGO, - false => return org_mode & S_IRWXUGO, + if is_explicit_no_preserve_mode { + return MODE_RW_UGO; + } else { + return org_mode & S_IRWXUGO; }; } } @@ -1984,6 +2521,7 @@ fn handle_no_preserve_mode(options: &Options, org_mode: u32) -> u32 { /// Copy the file from `source` to `dest` either using the normal `fs::copy` or a /// copy-on-write scheme if --reflink is specified and the filesystem supports it. +#[allow(clippy::too_many_arguments)] fn copy_helper( source: &Path, dest: &Path, @@ -1992,6 +2530,7 @@ fn copy_helper( source_is_symlink: bool, source_is_fifo: bool, symlinked_files: &mut HashSet, + #[cfg(unix)] source_is_stream: bool, ) -> CopyResult<()> { if options.parents { let parent = dest.parent().unwrap_or(dest); @@ -2002,14 +2541,9 @@ fn copy_helper( return Err(Error::NotADirectory(dest.to_path_buf())); } - if source.as_os_str() == "/dev/null" { - /* workaround a limitation of fs::copy - * https://github.com/rust-lang/rust/issues/79390 - */ - File::create(dest).context(dest.display().to_string())?; - } else if source_is_fifo && options.recursive && !options.copy_contents { + if source_is_fifo && options.recursive && !options.copy_contents { #[cfg(unix)] - copy_fifo(dest, options.overwrite)?; + copy_fifo(dest, options.overwrite, options.debug)?; } else if source_is_symlink { copy_link(source, dest, symlinked_files)?; } else { @@ -2019,8 +2553,10 @@ fn copy_helper( options.reflink_mode, options.sparse_mode, context, - #[cfg(any(target_os = "linux", target_os = "android", target_os = "macos"))] + #[cfg(unix)] source_is_fifo, + #[cfg(unix)] + source_is_stream, )?; if !options.attributes_only && options.debug { @@ -2034,18 +2570,13 @@ fn copy_helper( // "Copies" a FIFO by creating a new one. This workaround is because Rust's // built-in fs::copy does not handle FIFOs (see rust-lang/rust/issues/79390). #[cfg(unix)] -fn copy_fifo(dest: &Path, overwrite: OverwriteMode) -> CopyResult<()> { +fn copy_fifo(dest: &Path, overwrite: OverwriteMode, debug: bool) -> CopyResult<()> { if dest.exists() { - overwrite.verify(dest)?; + overwrite.verify(dest, debug)?; fs::remove_file(dest)?; } - let name = CString::new(dest.as_os_str().as_bytes()).unwrap(); - let err = unsafe { mkfifo(name.as_ptr(), 0o666) }; - if err == -1 { - return Err(format!("cannot create fifo {}: File exists", dest.quote()).into()); - } - Ok(()) + make_fifo(dest).map_err(|_| format!("cannot create fifo {}: File exists", dest.quote()).into()) } fn copy_link( @@ -2060,7 +2591,7 @@ fn copy_link( if dest.is_symlink() || dest.is_file() { fs::remove_file(dest)?; } - symlink_file(&link, dest, &context_for(&link, dest), symlinked_files) + symlink_file(&link, dest, symlinked_files) } /// Generate an error message if `target` is not the correct `target_type` @@ -2133,7 +2664,7 @@ fn disk_usage_directory(p: &Path) -> io::Result { #[cfg(test)] mod tests { - use crate::{aligned_ancestors, localize_to_target}; + use crate::{Attributes, Preserve, aligned_ancestors, localize_to_target}; use std::path::Path; #[test] @@ -2155,4 +2686,52 @@ mod tests { ]; assert_eq!(actual, expected); } + #[test] + fn test_diff_attrs() { + assert_eq!( + Attributes::ALL.diff(&Attributes { + context: Preserve::Yes { required: true }, + xattr: Preserve::Yes { required: true }, + ..Attributes::ALL + }), + Attributes { + #[cfg(unix)] + ownership: Preserve::No { explicit: true }, + mode: Preserve::No { explicit: true }, + timestamps: Preserve::No { explicit: true }, + context: Preserve::No { explicit: true }, + links: Preserve::No { explicit: true }, + xattr: Preserve::No { explicit: true } + } + ); + assert_eq!( + Attributes { + context: Preserve::Yes { required: true }, + xattr: Preserve::Yes { required: true }, + ..Attributes::ALL + } + .diff(&Attributes::NONE), + Attributes { + context: Preserve::Yes { required: true }, + xattr: Preserve::Yes { required: true }, + ..Attributes::ALL + } + ); + assert_eq!( + Attributes::NONE.diff(&Attributes { + context: Preserve::Yes { required: true }, + xattr: Preserve::Yes { required: true }, + ..Attributes::ALL + }), + Attributes { + #[cfg(unix)] + ownership: Preserve::No { explicit: true }, + mode: Preserve::No { explicit: true }, + timestamps: Preserve::No { explicit: true }, + context: Preserve::No { explicit: true }, + links: Preserve::No { explicit: true }, + xattr: Preserve::No { explicit: true } + } + ); + } } diff --git a/src/uu/cp/src/platform/linux.rs b/src/uu/cp/src/platform/linux.rs index 674e66ea575..9bf257f8276 100644 --- a/src/uu/cp/src/platform/linux.rs +++ b/src/uu/cp/src/platform/linux.rs @@ -2,12 +2,17 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore ficlone reflink ftruncate pwrite fiemap +// spell-checker:ignore ficlone reflink ftruncate pwrite fiemap lseek + +use libc::{SEEK_DATA, SEEK_HOLE}; use std::fs::{File, OpenOptions}; use std::io::Read; -use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::fs::FileExt; +use std::os::unix::fs::MetadataExt; +use std::os::unix::fs::{FileTypeExt, OpenOptionsExt}; use std::os::unix::io::AsRawFd; use std::path::Path; +use uucore::buf_copy; use quick_error::ResultExt; @@ -15,15 +20,6 @@ use uucore::mode::get_umask; use crate::{CopyDebug, CopyResult, OffloadReflinkDebug, ReflinkMode, SparseDebug, SparseMode}; -// From /usr/include/linux/fs.h: -// #define FICLONE _IOW(0x94, 9, int) -// Use a macro as libc::ioctl expects u32 or u64 depending on the arch -macro_rules! FICLONE { - () => { - 0x40049409 - }; -} - /// The fallback behavior for [`clone`] on failed system call. #[derive(Clone, Copy)] enum CloneFallback { @@ -32,6 +28,25 @@ enum CloneFallback { /// Use [`std::fs::copy`]. FSCopy, + + /// Use sparse_copy + SparseCopy, + + /// Use sparse_copy_without_hole + SparseCopyWithoutHole, +} + +/// Type of method used for copying files +#[derive(Clone, Copy)] +enum CopyMethod { + /// Do a sparse copy + SparseCopy, + /// Use [`std::fs::copy`]. + FSCopy, + /// Default (can either be sparse_copy or FSCopy) + Default, + /// Use sparse_copy_without_hole + SparseCopyWithoutHole, } /// Use the Linux `ioctl_ficlone` API to do a copy-on-write clone. @@ -46,24 +61,126 @@ where let dst_file = File::create(&dest)?; let src_fd = src_file.as_raw_fd(); let dst_fd = dst_file.as_raw_fd(); - let result = unsafe { libc::ioctl(dst_fd, FICLONE!(), src_fd) }; + // Using .try_into().unwrap() is required as glibc, musl & android all have different type for ioctl() + #[allow(clippy::unnecessary_fallible_conversions)] + let result = unsafe { + libc::ioctl( + dst_fd, + linux_raw_sys::ioctl::FICLONE.try_into().unwrap(), + src_fd, + ) + }; if result == 0 { return Ok(()); } match fallback { CloneFallback::Error => Err(std::io::Error::last_os_error()), CloneFallback::FSCopy => std::fs::copy(source, dest).map(|_| ()), + CloneFallback::SparseCopy => sparse_copy(source, dest), + CloneFallback::SparseCopyWithoutHole => sparse_copy_without_hole(source, dest), } } +/// Checks whether a file contains any non null bytes i.e. any byte != 0x0 +/// This function returns a tuple of (bool, u64, u64) signifying a tuple of (whether a file has +/// data, its size, no of blocks it has allocated in disk) +#[cfg(any(target_os = "linux", target_os = "android"))] +fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { + let mut src_file = File::open(source)?; + let metadata = src_file.metadata()?; + + let size = metadata.size(); + let blocks = metadata.blocks(); + // checks edge case of virtual files in /proc which have a size of zero but contains data + if size == 0 { + let mut buf: Vec = vec![0; metadata.blksize() as usize]; // Directly use metadata.blksize() + let _ = src_file.read(&mut buf)?; + return Ok((buf.iter().any(|&x| x != 0x0), size, 0)); + } + + let src_fd = src_file.as_raw_fd(); + + let result = unsafe { libc::lseek(src_fd, 0, SEEK_DATA) }; + + match result { + -1 => Ok((false, size, blocks)), // No data found or end of file + _ if result >= 0 => Ok((true, size, blocks)), // Data found + _ => Err(std::io::Error::last_os_error()), + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +/// Checks whether a file is sparse i.e. it contains holes, uses the crude heuristic blocks < size / 512 +/// Reference:`` +fn check_sparse_detection(source: &Path) -> Result { + let src_file = File::open(source)?; + let metadata = src_file.metadata()?; + let size = metadata.size(); + let blocks = metadata.blocks(); + + if blocks < size / 512 { + return Ok(true); + } + Ok(false) +} + +/// Optimized sparse_copy, doesn't create holes for large sequences of zeros in non sparse_files +/// Used when --sparse=auto +#[cfg(any(target_os = "linux", target_os = "android"))] +fn sparse_copy_without_hole

(source: P, dest: P) -> std::io::Result<()> +where + P: AsRef, +{ + let src_file = File::open(source)?; + let dst_file = File::create(dest)?; + let dst_fd = dst_file.as_raw_fd(); + + let size = src_file.metadata()?.size(); + if unsafe { libc::ftruncate(dst_fd, size.try_into().unwrap()) } < 0 { + return Err(std::io::Error::last_os_error()); + } + let src_fd = src_file.as_raw_fd(); + let mut current_offset: isize = 0; + // Maximize the data read at once to 16 MiB to avoid memory hogging with large files + // 16 MiB chunks should saturate an SSD + let step = std::cmp::min(size, 16 * 1024 * 1024) as usize; + let mut buf: Vec = vec![0x0; step]; + loop { + let result = unsafe { libc::lseek(src_fd, current_offset.try_into().unwrap(), SEEK_DATA) } + .try_into() + .unwrap(); + + current_offset = result; + let hole: isize = + unsafe { libc::lseek(src_fd, current_offset.try_into().unwrap(), SEEK_HOLE) } + .try_into() + .unwrap(); + if result == -1 || hole == -1 { + break; + } + if result <= -2 || hole <= -2 { + return Err(std::io::Error::last_os_error()); + } + let len: isize = hole - current_offset; + // Read and write data in chunks of `step` while reusing the same buffer + for i in (0..len).step_by(step) { + // Ensure we don't read past the end of the file or the start of the next hole + let read_len = std::cmp::min((len - i) as usize, step); + let buf = &mut buf[..read_len]; + src_file.read_exact_at(buf, (current_offset + i) as u64)?; + dst_file.write_all_at(buf, (current_offset + i) as u64)?; + } + current_offset = hole; + } + Ok(()) +} /// Perform a sparse copy from one file to another. +/// Creates a holes for large sequences of zeros in non_sparse_files, used for --sparse=always #[cfg(any(target_os = "linux", target_os = "android"))] fn sparse_copy

(source: P, dest: P) -> std::io::Result<()> where P: AsRef, { - use std::os::unix::prelude::MetadataExt; - let mut src_file = File::open(source)?; let dst_file = File::create(dest)?; let dst_fd = dst_file.as_raw_fd(); @@ -82,23 +199,30 @@ where // https://www.kernel.org/doc/html/latest/filesystems/fiemap.html while current_offset < size { let this_read = src_file.read(&mut buf)?; + let buf = &buf[..this_read]; if buf.iter().any(|&x| x != 0) { - unsafe { - libc::pwrite( - dst_fd, - buf.as_ptr() as *const libc::c_void, - this_read, - current_offset.try_into().unwrap(), - ) - }; + dst_file.write_all_at(buf, current_offset.try_into().unwrap())?; } current_offset += this_read; } Ok(()) } -/// Copy the contents of the given source FIFO to the given file. -fn copy_fifo_contents

(source: P, dest: P) -> std::io::Result +#[cfg(any(target_os = "linux", target_os = "android"))] +/// Checks whether an existing destination is a fifo +fn check_dest_is_fifo(dest: &Path) -> bool { + // If our destination file exists and its a fifo , we do a standard copy . + let file_type = std::fs::metadata(dest); + match file_type { + Ok(f) => f.file_type().is_fifo(), + + _ => false, + } +} + +/// Copy the contents of a stream from `source` to `dest`. The `if_fifo` argument is used to +/// determine if we need to modify the file's attributes before and after copying. +fn copy_stream

(source: P, dest: P, is_fifo: bool) -> std::io::Result where P: AsRef, { @@ -127,8 +251,14 @@ where .write(true) .mode(mode) .open(&dest)?; - let num_bytes_copied = std::io::copy(&mut src_file, &mut dst_file)?; - dst_file.set_permissions(src_file.metadata()?.permissions())?; + + let num_bytes_copied = buf_copy::copy_stream(&mut src_file, &mut dst_file) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::Other))?; + + if is_fifo { + dst_file.set_permissions(src_file.metadata()?.permissions())?; + } + Ok(num_bytes_copied) } @@ -145,41 +275,125 @@ pub(crate) fn copy_on_write( sparse_mode: SparseMode, context: &str, source_is_fifo: bool, + source_is_stream: bool, ) -> CopyResult { let mut copy_debug = CopyDebug { offload: OffloadReflinkDebug::Unknown, reflink: OffloadReflinkDebug::Unsupported, sparse_detection: SparseDebug::No, }; - let result = match (reflink_mode, sparse_mode) { (ReflinkMode::Never, SparseMode::Always) => { copy_debug.sparse_detection = SparseDebug::Zeros; - copy_debug.offload = OffloadReflinkDebug::Avoided; + // Default SparseDebug val for SparseMode::Always copy_debug.reflink = OffloadReflinkDebug::No; - sparse_copy(source, dest) + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_never_sparse_always(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; + } + + match copy_method { + CopyMethod::FSCopy => std::fs::copy(source, dest).map(|_| ()), + _ => sparse_copy(source, dest), + } + } } - (ReflinkMode::Never, _) => { - copy_debug.sparse_detection = SparseDebug::No; + (ReflinkMode::Never, SparseMode::Never) => { copy_debug.reflink = OffloadReflinkDebug::No; - std::fs::copy(source, dest).map(|_| ()) + + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let result = handle_reflink_never_sparse_never(source); + if let Ok(debug) = result { + copy_debug = debug; + } + std::fs::copy(source, dest).map(|_| ()) + } + } + (ReflinkMode::Never, SparseMode::Auto) => { + copy_debug.reflink = OffloadReflinkDebug::No; + + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_never_sparse_auto(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; + } + + match copy_method { + CopyMethod::SparseCopyWithoutHole => sparse_copy_without_hole(source, dest), + _ => std::fs::copy(source, dest).map(|_| ()), + } + } } (ReflinkMode::Auto, SparseMode::Always) => { - copy_debug.offload = OffloadReflinkDebug::Avoided; - copy_debug.sparse_detection = SparseDebug::Zeros; - copy_debug.reflink = OffloadReflinkDebug::Unsupported; - sparse_copy(source, dest) + copy_debug.sparse_detection = SparseDebug::Zeros; // Default SparseDebug val for + // SparseMode::Always + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_auto_sparse_always(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; + } + + match copy_method { + CopyMethod::FSCopy => clone(source, dest, CloneFallback::FSCopy), + _ => clone(source, dest, CloneFallback::SparseCopy), + } + } } - (ReflinkMode::Auto, _) => { - copy_debug.sparse_detection = SparseDebug::No; - copy_debug.reflink = OffloadReflinkDebug::Unsupported; - if source_is_fifo { - copy_fifo_contents(source, dest).map(|_| ()) + (ReflinkMode::Auto, SparseMode::Never) => { + copy_debug.reflink = OffloadReflinkDebug::No; + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) } else { + let result = handle_reflink_auto_sparse_never(source); + if let Ok(debug) = result { + copy_debug = debug; + } + clone(source, dest, CloneFallback::FSCopy) } } + (ReflinkMode::Auto, SparseMode::Auto) => { + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Unsupported; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_auto_sparse_auto(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; + } + + match copy_method { + CopyMethod::SparseCopyWithoutHole => { + clone(source, dest, CloneFallback::SparseCopyWithoutHole) + } + _ => clone(source, dest, CloneFallback::FSCopy), + } + } + } + (ReflinkMode::Always, SparseMode::Auto) => { copy_debug.sparse_detection = SparseDebug::No; copy_debug.reflink = OffloadReflinkDebug::Yes; @@ -187,9 +401,217 @@ pub(crate) fn copy_on_write( clone(source, dest, CloneFallback::Error) } (ReflinkMode::Always, _) => { - return Err("`--reflink=always` can be used only with --sparse=auto".into()) + return Err("`--reflink=always` can be used only with --sparse=auto".into()); } }; result.context(context)?; Ok(copy_debug) } + +/// Handles debug results when flags are "--reflink=auto" and "--sparse=always" and specifies what +/// type of copy should be used +fn handle_reflink_auto_sparse_always( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::Unsupported, + sparse_detection: SparseDebug::Zeros, + }; + let mut copy_method = CopyMethod::Default; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + match (sparse_flag, data_flag, blocks) { + (true, true, 0) => { + // Handling funny files with 0 block allocation but has data + // in it + copy_method = CopyMethod::FSCopy; + copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; + } + (false, true, 0) => copy_method = CopyMethod::FSCopy, + + (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, + (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, + + (true, false, _) => copy_debug.sparse_detection = SparseDebug::SeekHole, + + (_, _, _) => (), + } + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + Ok((copy_debug, copy_method)) +} + +/// Handles debug results when flags are "--reflink=auto" and "--sparse=auto" and specifies what +/// type of copy should be used +fn handle_reflink_never_sparse_never(source: &Path) -> Result { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::No, + }; + let (data_flag, size, _blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if sparse_flag { + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + Ok(copy_debug) +} + +/// Handles debug results when flags are "--reflink=auto" and "--sparse=never", files will be copied +/// through cloning them with fallback switching to std::fs::copy +fn handle_reflink_auto_sparse_never(source: &Path) -> Result { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::No, + }; + + let (data_flag, size, _blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if sparse_flag { + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + Ok(copy_debug) +} + +/// Handles debug results when flags are "--reflink=auto" and "--sparse=auto" and specifies what +/// type of copy should be used +fn handle_reflink_auto_sparse_auto( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::Unsupported, + sparse_detection: SparseDebug::No, + }; + + let mut copy_method = CopyMethod::Default; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if (data_flag && size != 0) || (size > 0 && size < 512) { + copy_debug.offload = OffloadReflinkDebug::Yes; + } + + if data_flag && size == 0 { + // Handling /proc/ files + copy_debug.offload = OffloadReflinkDebug::Unsupported; + } + if sparse_flag { + if blocks == 0 && data_flag { + // Handling other "virtual" files + copy_debug.offload = OffloadReflinkDebug::Unsupported; + + copy_method = CopyMethod::FSCopy; // Doing a standard copy for the virtual files + } else { + copy_method = CopyMethod::SparseCopyWithoutHole; + } // Since sparse_flag is true, sparse_detection shall be SeekHole for any non virtual + // regular sparse file and the file will be sparsely copied + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + Ok((copy_debug, copy_method)) +} + +/// Handles debug results when flags are "--reflink=never" and "--sparse=auto" and specifies what +/// type of copy should be used +fn handle_reflink_never_sparse_auto( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::No, + }; + + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + let mut copy_method = CopyMethod::Default; + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + + if sparse_flag { + if blocks == 0 && data_flag { + copy_method = CopyMethod::FSCopy; // Handles virtual files which have size > 0 but no + // disk allocation + } else { + copy_method = CopyMethod::SparseCopyWithoutHole; // Handles regular sparse-files + } + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + Ok((copy_debug, copy_method)) +} + +/// Handles debug results when flags are "--reflink=never" and "--sparse=always" and specifies what +/// type of copy should be used +fn handle_reflink_never_sparse_always( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::Zeros, + }; + let mut copy_method = CopyMethod::SparseCopy; + + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + match (sparse_flag, data_flag, blocks) { + (true, true, 0) => { + // Handling funny files with 0 block allocation but has data + // in it, e.g. files in /sys and other virtual files + copy_method = CopyMethod::FSCopy; + copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; + } + (false, true, 0) => copy_method = CopyMethod::FSCopy, // Handling data containing zero sized + // files in /proc + (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, // Handles files + // with 0 blocks allocated in disk and + (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, // Any + // sparse_files with data in it will display SeekHoleZeros + (true, false, _) => { + copy_debug.offload = OffloadReflinkDebug::Unknown; + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + (_, _, _) => (), + } + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + + Ok((copy_debug, copy_method)) +} diff --git a/src/uu/cp/src/platform/macos.rs b/src/uu/cp/src/platform/macos.rs index 77bdbbbdb83..ee5ddca5463 100644 --- a/src/uu/cp/src/platform/macos.rs +++ b/src/uu/cp/src/platform/macos.rs @@ -4,12 +4,14 @@ // file that was distributed with this source code. // spell-checker:ignore reflink use std::ffi::CString; -use std::fs::{self, File}; -use std::io; +use std::fs::{self, File, OpenOptions}; use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use quick_error::ResultExt; +use uucore::buf_copy; +use uucore::mode::get_umask; use crate::{CopyDebug, CopyResult, OffloadReflinkDebug, ReflinkMode, SparseDebug, SparseMode}; @@ -24,6 +26,7 @@ pub(crate) fn copy_on_write( sparse_mode: SparseMode, context: &str, source_is_fifo: bool, + source_is_stream: bool, ) -> CopyResult { if sparse_mode != SparseMode::Auto { return Err("--sparse is only supported on linux".to_string().into()); @@ -64,7 +67,7 @@ pub(crate) fn copy_on_write( // clonefile(2) fails if the destination exists. Remove it and try again. Do not // bother to check if removal worked because we're going to try to clone again. // first lets make sure the dest file is not read only - if fs::metadata(dest).map_or(false, |md| !md.permissions().readonly()) { + if fs::metadata(dest).is_ok_and(|md| !md.permissions().readonly()) { // remove and copy again // TODO: rewrite this to better match linux behavior // linux first opens the source file and destination file then uses the file @@ -81,14 +84,27 @@ pub(crate) fn copy_on_write( // support COW). match reflink_mode { ReflinkMode::Always => { - return Err(format!("failed to clone {source:?} from {dest:?}: {error}").into()) + return Err(format!("failed to clone {source:?} from {dest:?}: {error}").into()); } _ => { copy_debug.reflink = OffloadReflinkDebug::Yes; - if source_is_fifo { + if source_is_stream { let mut src_file = File::open(source)?; - let mut dst_file = File::create(dest)?; - io::copy(&mut src_file, &mut dst_file).context(context)? + let mode = 0o622 & !get_umask(); + let mut dst_file = OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest)?; + + let context = buf_copy::copy_stream(&mut src_file, &mut dst_file) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::Other)) + .context(context)?; + + if source_is_fifo { + dst_file.set_permissions(src_file.metadata()?.permissions())?; + } + context } else { fs::copy(source, dest).context(context)? } diff --git a/src/uu/cp/src/platform/mod.rs b/src/uu/cp/src/platform/mod.rs index c7942706868..2071e928f41 100644 --- a/src/uu/cp/src/platform/mod.rs +++ b/src/uu/cp/src/platform/mod.rs @@ -2,6 +2,18 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. + +#[cfg(all( + unix, + not(any(target_os = "macos", target_os = "linux", target_os = "android")) +))] +mod other_unix; +#[cfg(all( + unix, + not(any(target_os = "macos", target_os = "linux", target_os = "android")) +))] +pub(crate) use self::other_unix::copy_on_write; + #[cfg(target_os = "macos")] mod macos; #[cfg(target_os = "macos")] @@ -12,7 +24,13 @@ mod linux; #[cfg(any(target_os = "linux", target_os = "android"))] pub(crate) use self::linux::copy_on_write; -#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))] +#[cfg(not(any( + unix, + any(target_os = "macos", target_os = "linux", target_os = "android") +)))] mod other; -#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))] +#[cfg(not(any( + unix, + any(target_os = "macos", target_os = "linux", target_os = "android") +)))] pub(crate) use self::other::copy_on_write; diff --git a/src/uu/cp/src/platform/other_unix.rs b/src/uu/cp/src/platform/other_unix.rs new file mode 100644 index 00000000000..aa8fed3fab1 --- /dev/null +++ b/src/uu/cp/src/platform/other_unix.rs @@ -0,0 +1,62 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore reflink +use std::fs::{self, File, OpenOptions}; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; + +use quick_error::ResultExt; +use uucore::buf_copy; +use uucore::mode::get_umask; + +use crate::{CopyDebug, CopyResult, OffloadReflinkDebug, ReflinkMode, SparseDebug, SparseMode}; + +/// Copies `source` to `dest` for systems without copy-on-write +pub(crate) fn copy_on_write( + source: &Path, + dest: &Path, + reflink_mode: ReflinkMode, + sparse_mode: SparseMode, + context: &str, + source_is_fifo: bool, + source_is_stream: bool, +) -> CopyResult { + if reflink_mode != ReflinkMode::Never { + return Err("--reflink is only supported on linux and macOS" + .to_string() + .into()); + } + if sparse_mode != SparseMode::Auto { + return Err("--sparse is only supported on linux".to_string().into()); + } + let copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unsupported, + reflink: OffloadReflinkDebug::Unsupported, + sparse_detection: SparseDebug::Unsupported, + }; + + if source_is_stream { + let mut src_file = File::open(source)?; + let mode = 0o622 & !get_umask(); + let mut dst_file = OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest)?; + + buf_copy::copy_stream(&mut src_file, &mut dst_file) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::Other)) + .context(context)?; + + if source_is_fifo { + dst_file.set_permissions(src_file.metadata()?.permissions())?; + } + return Ok(copy_debug); + } + + fs::copy(source, dest).context(context)?; + + Ok(copy_debug) +} diff --git a/src/uu/csplit/Cargo.toml b/src/uu/csplit/Cargo.toml index 5e2f310cb51..508656f681d 100644 --- a/src/uu/csplit/Cargo.toml +++ b/src/uu/csplit/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_csplit" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "csplit ~ (uutils) Output pieces of FILE separated by PATTERN(s) to files 'xx00', 'xx01', ..., and output byte counts of each piece to standard output" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/ls" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/csplit.rs" @@ -18,7 +21,7 @@ path = "src/csplit.rs" clap = { workspace = true } thiserror = { workspace = true } regex = { workspace = true } -uucore = { workspace = true, features = ["entries", "fs"] } +uucore = { workspace = true, features = ["entries", "fs", "format"] } [[bin]] name = "csplit" diff --git a/src/uu/csplit/src/csplit.rs b/src/uu/csplit/src/csplit.rs index 00bebbf4dcb..621823aebba 100644 --- a/src/uu/csplit/src/csplit.rs +++ b/src/uu/csplit/src/csplit.rs @@ -2,22 +2,21 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -#![crate_name = "uu_csplit"] // spell-checker:ignore rustdoc #![allow(rustdoc::private_intra_doc_links)] use std::cmp::Ordering; -use std::io::{self, BufReader}; +use std::io::{self, BufReader, ErrorKind}; use std::{ - fs::{remove_file, File}, + fs::{File, remove_file}, io::{BufRead, BufWriter, Write}, }; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; use regex::Regex; use uucore::display::Quotable; use uucore::error::{FromIo, UResult}; -use uucore::{crash_if_err, format_usage, help_about, help_section, help_usage}; +use uucore::{format_usage, help_about, help_section, help_usage}; mod csplit_error; mod patterns; @@ -44,7 +43,7 @@ mod options { /// Command line options for csplit. pub struct CsplitOptions { - split_name: crate::SplitName, + split_name: SplitName, keep_files: bool, quiet: bool, elide_empty_files: bool, @@ -52,25 +51,51 @@ pub struct CsplitOptions { } impl CsplitOptions { - fn new(matches: &ArgMatches) -> Self { + fn new(matches: &ArgMatches) -> Result { let keep_files = matches.get_flag(options::KEEP_FILES); let quiet = matches.get_flag(options::QUIET); let elide_empty_files = matches.get_flag(options::ELIDE_EMPTY_FILES); let suppress_matched = matches.get_flag(options::SUPPRESS_MATCHED); - Self { - split_name: crash_if_err!( - 1, - SplitName::new( - matches.get_one::(options::PREFIX).cloned(), - matches.get_one::(options::SUFFIX_FORMAT).cloned(), - matches.get_one::(options::DIGITS).cloned() - ) - ), + Ok(Self { + split_name: SplitName::new( + matches.get_one::(options::PREFIX).cloned(), + matches.get_one::(options::SUFFIX_FORMAT).cloned(), + matches.get_one::(options::DIGITS).cloned(), + )?, keep_files, quiet, elide_empty_files, suppress_matched, + }) + } +} + +pub struct LinesWithNewlines { + inner: T, +} + +impl LinesWithNewlines { + fn new(s: T) -> Self { + Self { inner: s } + } +} + +impl Iterator for LinesWithNewlines { + type Item = io::Result; + + fn next(&mut self) -> Option { + fn ret(v: Vec) -> io::Result { + String::from_utf8(v).map_err(|_| { + io::Error::new(ErrorKind::InvalidData, "stream did not contain valid UTF-8") + }) + } + + let mut v = Vec::new(); + match self.inner.read_until(b'\n', &mut v) { + Ok(0) => None, + Ok(_) => Some(ret(v)), + Err(e) => Some(Err(e)), } } } @@ -87,27 +112,29 @@ impl CsplitOptions { /// - [`CsplitError::MatchNotFound`] if no line matched a regular expression. /// - [`CsplitError::MatchNotFoundOnRepetition`], like previous but after applying the pattern /// more than once. -pub fn csplit( - options: &CsplitOptions, - patterns: Vec, - input: T, -) -> Result<(), CsplitError> +pub fn csplit(options: &CsplitOptions, patterns: &[String], input: T) -> Result<(), CsplitError> where T: BufRead, { - let mut input_iter = InputSplitter::new(input.lines().enumerate()); + let enumerated_input_lines = LinesWithNewlines::new(input) + .map(|line| line.map_err_context(|| "read error".to_string())) + .enumerate(); + let mut input_iter = InputSplitter::new(enumerated_input_lines); let mut split_writer = SplitWriter::new(options); + let patterns: Vec = patterns::get_patterns(patterns)?; let ret = do_csplit(&mut split_writer, patterns, &mut input_iter); - // consume the rest - input_iter.rewind_buffer(); - if let Some((_, line)) = input_iter.next() { - split_writer.new_writer()?; - split_writer.writeln(&line?)?; - for (_, line) in input_iter { + // consume the rest, unless there was an error + if ret.is_ok() { + input_iter.rewind_buffer(); + if let Some((_, line)) = input_iter.next() { + split_writer.new_writer()?; split_writer.writeln(&line?)?; + for (_, line) in input_iter { + split_writer.writeln(&line?)?; + } + split_writer.finish_split(); } - split_writer.finish_split(); } // delete files on error by default if ret.is_err() && !options.keep_files { @@ -122,7 +149,7 @@ fn do_csplit( input_iter: &mut InputSplitter, ) -> Result<(), CsplitError> where - I: Iterator)>, + I: Iterator)>, { // split the file based on patterns for pattern in patterns { @@ -199,16 +226,21 @@ struct SplitWriter<'a> { dev_null: bool, } -impl<'a> Drop for SplitWriter<'a> { +impl Drop for SplitWriter<'_> { fn drop(&mut self) { if self.options.elide_empty_files && self.size == 0 { let file_name = self.options.split_name.get(self.counter); - remove_file(file_name).expect("Failed to elide split"); + // In the case of `echo a | csplit -z - %a%1`, the file + // `xx00` does not exist because the positive offset + // advanced past the end of the input. Since there is no + // file to remove in that case, `remove_file` would return + // an error, so we just ignore it. + let _ = remove_file(file_name); } } } -impl<'a> SplitWriter<'a> { +impl SplitWriter<'_> { fn new(options: &CsplitOptions) -> SplitWriter { SplitWriter { options, @@ -239,7 +271,7 @@ impl<'a> SplitWriter<'a> { self.dev_null = true; } - /// Writes the line to the current split, appending a newline character. + /// Writes the line to the current split. /// If [`self.dev_null`] is true, then the line is discarded. /// /// # Errors @@ -251,8 +283,7 @@ impl<'a> SplitWriter<'a> { Some(ref mut current_writer) => { let bytes = line.as_bytes(); current_writer.write_all(bytes)?; - current_writer.write_all(b"\n")?; - self.size += bytes.len() + 1; + self.size += bytes.len(); } None => panic!("trying to write to a split that was not created"), } @@ -310,18 +341,18 @@ impl<'a> SplitWriter<'a> { input_iter: &mut InputSplitter, ) -> Result<(), CsplitError> where - I: Iterator)>, + I: Iterator)>, { input_iter.rewind_buffer(); input_iter.set_size_of_buffer(1); let mut ret = Err(CsplitError::LineOutOfRange(pattern_as_str.to_string())); while let Some((ln, line)) = input_iter.next() { - let l = line?; + let line = line?; match n.cmp(&(&ln + 1)) { Ordering::Less => { assert!( - input_iter.add_line_to_buffer(ln, l).is_none(), + input_iter.add_line_to_buffer(ln, line).is_none(), "the buffer is big enough to contain 1 line" ); ret = Ok(()); @@ -330,7 +361,7 @@ impl<'a> SplitWriter<'a> { Ordering::Equal => { assert!( self.options.suppress_matched - || input_iter.add_line_to_buffer(ln, l).is_none(), + || input_iter.add_line_to_buffer(ln, line).is_none(), "the buffer is big enough to contain 1 line" ); ret = Ok(()); @@ -338,7 +369,7 @@ impl<'a> SplitWriter<'a> { } Ordering::Greater => (), } - self.writeln(&l)?; + self.writeln(&line)?; } self.finish_split(); ret @@ -353,7 +384,7 @@ impl<'a> SplitWriter<'a> { /// In addition to errors reading/writing from/to a file, the following errors may be returned: /// - if no line matched, an [`CsplitError::MatchNotFound`]. /// - if there are not enough lines to accommodate the offset, an - /// [`CsplitError::LineOutOfRange`]. + /// [`CsplitError::LineOutOfRange`]. #[allow(clippy::cognitive_complexity)] fn do_to_match( &mut self, @@ -363,7 +394,7 @@ impl<'a> SplitWriter<'a> { input_iter: &mut InputSplitter, ) -> Result<(), CsplitError> where - I: Iterator)>, + I: Iterator)>, { if offset >= 0 { // The offset is zero or positive, no need for a buffer on the lines read. @@ -375,18 +406,27 @@ impl<'a> SplitWriter<'a> { input_iter.set_size_of_buffer(1); while let Some((ln, line)) = input_iter.next() { - let l = line?; - if regex.is_match(&l) { + let line = line?; + let l = line + .strip_suffix("\r\n") + .unwrap_or_else(|| line.strip_suffix('\n').unwrap_or(&line)); + if regex.is_match(l) { + let mut next_line_suppress_matched = false; match (self.options.suppress_matched, offset) { // no offset, add the line to the next split (false, 0) => { assert!( - input_iter.add_line_to_buffer(ln, l).is_none(), + input_iter.add_line_to_buffer(ln, line).is_none(), "the buffer is big enough to contain 1 line" ); } // a positive offset, some more lines need to be added to the current split - (false, _) => self.writeln(&l)?, + (false, _) => self.writeln(&line)?, + // suppress matched option true, but there is a positive offset, so the line is printed + (true, 1..) => { + next_line_suppress_matched = true; + self.writeln(&line)?; + } _ => (), }; offset -= 1; @@ -407,9 +447,14 @@ impl<'a> SplitWriter<'a> { offset -= 1; } self.finish_split(); + + // if we have to suppress one line after we take the next and do nothing + if next_line_suppress_matched { + input_iter.next(); + } return Ok(()); } - self.writeln(&l)?; + self.writeln(&line)?; } } else { // With a negative offset we use a buffer to keep the lines within the offset. @@ -420,26 +465,35 @@ impl<'a> SplitWriter<'a> { let offset_usize = -offset as usize; input_iter.set_size_of_buffer(offset_usize); while let Some((ln, line)) = input_iter.next() { - let l = line?; - if regex.is_match(&l) { + let line = line?; + let l = line + .strip_suffix("\r\n") + .unwrap_or_else(|| line.strip_suffix('\n').unwrap_or(&line)); + if regex.is_match(l) { for line in input_iter.shrink_buffer_to_size() { self.writeln(&line)?; } - if !self.options.suppress_matched { + if self.options.suppress_matched { + // since offset_usize is for sure greater than 0 + // the first element of the buffer should be removed and this + // line inserted to be coherent with GNU implementation + input_iter.add_line_to_buffer(ln, line); + } else { // add 1 to the buffer size to make place for the matched line input_iter.set_size_of_buffer(offset_usize + 1); assert!( - input_iter.add_line_to_buffer(ln, l).is_none(), + input_iter.add_line_to_buffer(ln, line).is_none(), "should be big enough to hold every lines" ); } + self.finish_split(); if input_iter.buffer_len() < offset_usize { return Err(CsplitError::LineOutOfRange(pattern_as_str.to_string())); } return Ok(()); } - if let Some(line) = input_iter.add_line_to_buffer(ln, l) { + if let Some(line) = input_iter.add_line_to_buffer(ln, line) { self.writeln(&line)?; } } @@ -458,7 +512,7 @@ impl<'a> SplitWriter<'a> { /// This is used to pass matching lines to the next split and to support patterns with a negative offset. struct InputSplitter where - I: Iterator)>, + I: Iterator)>, { iter: I, buffer: Vec<::Item>, @@ -471,7 +525,7 @@ where impl InputSplitter where - I: Iterator)>, + I: Iterator)>, { fn new(iter: I) -> Self { Self { @@ -535,7 +589,7 @@ where impl Iterator for InputSplitter where - I: Iterator)>, + I: Iterator)>, { type Item = ::Item; @@ -563,29 +617,23 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { .unwrap() .map(|s| s.to_string()) .collect(); - let patterns = patterns::get_patterns(&patterns[..])?; - let options = CsplitOptions::new(&matches); + let options = CsplitOptions::new(&matches)?; if file_name == "-" { let stdin = io::stdin(); - Ok(csplit(&options, patterns, stdin.lock())?) + Ok(csplit(&options, &patterns, stdin.lock())?) } else { let file = File::open(file_name) - .map_err_context(|| format!("cannot access {}", file_name.quote()))?; - let file_metadata = file - .metadata() - .map_err_context(|| format!("cannot access {}", file_name.quote()))?; - if !file_metadata.is_file() { - return Err(CsplitError::NotRegularFile(file_name.to_string()).into()); - } - Ok(csplit(&options, patterns, BufReader::new(file))?) + .map_err_context(|| format!("cannot open {} for reading", file_name.quote()))?; + Ok(csplit(&options, &patterns, BufReader::new(file))?) } } pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) + .args_override_self(true) .infer_long_args(true) .arg( Arg::new(options::SUFFIX_FORMAT) @@ -623,8 +671,9 @@ pub fn uu_app() -> Command { ) .arg( Arg::new(options::QUIET) - .short('s') + .short('q') .long(options::QUIET) + .visible_short_alias('s') .visible_alias("silent") .help("do not print counts of output file sizes") .action(ArgAction::SetTrue), @@ -645,7 +694,7 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::PATTERN) .hide(true) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) .required(true), ) .after_help(AFTER_HELP) diff --git a/src/uu/csplit/src/csplit_error.rs b/src/uu/csplit/src/csplit_error.rs index 1559a29f8bc..a8c0fd1af08 100644 --- a/src/uu/csplit/src/csplit_error.rs +++ b/src/uu/csplit/src/csplit_error.rs @@ -12,7 +12,7 @@ use uucore::error::UError; #[derive(Debug, Error)] pub enum CsplitError { #[error("IO error: {}", _0)] - IoError(io::Error), + IoError(#[from] io::Error), #[error("{}: line number out of range", ._0.quote())] LineOutOfRange(String), #[error("{}: line number out of range on repetition {}", ._0.quote(), _1)] @@ -21,7 +21,7 @@ pub enum CsplitError { MatchNotFound(String), #[error("{}: match not found on repetition {}", ._0.quote(), _1)] MatchNotFoundOnRepetition(String, usize), - #[error("line number must be greater than zero")] + #[error("0: line number must be greater than zero")] LineNumberIsZero, #[error("line number '{}' is smaller than preceding line number, {}", _0, _1)] LineNumberSmallerThanPrevious(usize, usize), @@ -35,16 +35,21 @@ pub enum CsplitError { SuffixFormatTooManyPercents, #[error("{} is not a regular file", ._0.quote())] NotRegularFile(String), + #[error("{}", _0)] + UError(Box), } -impl From for CsplitError { - fn from(error: io::Error) -> Self { - Self::IoError(error) +impl From> for CsplitError { + fn from(error: Box) -> Self { + Self::UError(error) } } impl UError for CsplitError { fn code(&self) -> i32 { - 1 + match self { + Self::UError(e) => e.code(), + _ => 1, + } } } diff --git a/src/uu/csplit/src/patterns.rs b/src/uu/csplit/src/patterns.rs index 6e7483b7f9f..bdf15b51197 100644 --- a/src/uu/csplit/src/patterns.rs +++ b/src/uu/csplit/src/patterns.rs @@ -25,14 +25,14 @@ pub enum Pattern { SkipToMatch(Regex, i32, ExecutePattern), } -impl ToString for Pattern { - fn to_string(&self) -> String { +impl std::fmt::Display for Pattern { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::UpToLine(n, _) => n.to_string(), - Self::UpToMatch(regex, 0, _) => format!("/{}/", regex.as_str()), - Self::UpToMatch(regex, offset, _) => format!("/{}/{:+}", regex.as_str(), offset), - Self::SkipToMatch(regex, 0, _) => format!("%{}%", regex.as_str()), - Self::SkipToMatch(regex, offset, _) => format!("%{}%{:+}", regex.as_str(), offset), + Self::UpToLine(n, _) => write!(f, "{n}"), + Self::UpToMatch(regex, 0, _) => write!(f, "/{}/", regex.as_str()), + Self::UpToMatch(regex, offset, _) => write!(f, "/{}/{offset:+}", regex.as_str()), + Self::SkipToMatch(regex, 0, _) => write!(f, "%{}%", regex.as_str()), + Self::SkipToMatch(regex, offset, _) => write!(f, "%{}%{offset:+}", regex.as_str()), } } } @@ -106,7 +106,7 @@ pub fn get_patterns(args: &[String]) -> Result, CsplitError> { fn extract_patterns(args: &[String]) -> Result, CsplitError> { let mut patterns = Vec::with_capacity(args.len()); let to_match_reg = - Regex::new(r"^(/(?P.+)/|%(?P.+)%)(?P[\+-]\d+)?$").unwrap(); + Regex::new(r"^(/(?P.+)/|%(?P.+)%)(?P[\+-]?\d+)?$").unwrap(); let execute_ntimes_reg = Regex::new(r"^\{(?P\d+)|\*\}$").unwrap(); let mut iter = args.iter().peekable(); @@ -168,7 +168,7 @@ fn validate_line_numbers(patterns: &[Pattern]) -> Result<(), CsplitError> { (_, 0) => Err(CsplitError::LineNumberIsZero), // two consecutive numbers should not be equal (n, m) if n == m => { - show_warning!("line number '{}' is the same as preceding line number", n); + show_warning!("line number '{n}' is the same as preceding line number"); Ok(n) } // a number cannot be greater than the one that follows @@ -219,14 +219,15 @@ mod tests { "{*}", "/test3.*end$/", "{4}", - "/test4.*end$/+3", - "/test5.*end$/-3", + "/test4.*end$/3", + "/test5.*end$/+3", + "/test6.*end$/-3", ] .into_iter() .map(|v| v.to_string()) .collect(); let patterns = get_patterns(input.as_slice()).unwrap(); - assert_eq!(patterns.len(), 5); + assert_eq!(patterns.len(), 6); match patterns.first() { Some(Pattern::UpToMatch(reg, 0, ExecutePattern::Times(1))) => { let parsed_reg = format!("{reg}"); @@ -256,12 +257,19 @@ mod tests { _ => panic!("expected UpToMatch pattern"), }; match patterns.get(4) { - Some(Pattern::UpToMatch(reg, -3, ExecutePattern::Times(1))) => { + Some(Pattern::UpToMatch(reg, 3, ExecutePattern::Times(1))) => { let parsed_reg = format!("{reg}"); assert_eq!(parsed_reg, "test5.*end$"); } _ => panic!("expected UpToMatch pattern"), }; + match patterns.get(5) { + Some(Pattern::UpToMatch(reg, -3, ExecutePattern::Times(1))) => { + let parsed_reg = format!("{reg}"); + assert_eq!(parsed_reg, "test6.*end$"); + } + _ => panic!("expected UpToMatch pattern"), + }; } #[test] @@ -273,14 +281,15 @@ mod tests { "{*}", "%test3.*end$%", "{4}", - "%test4.*end$%+3", - "%test5.*end$%-3", + "%test4.*end$%3", + "%test5.*end$%+3", + "%test6.*end$%-3", ] .into_iter() .map(|v| v.to_string()) .collect(); let patterns = get_patterns(input.as_slice()).unwrap(); - assert_eq!(patterns.len(), 5); + assert_eq!(patterns.len(), 6); match patterns.first() { Some(Pattern::SkipToMatch(reg, 0, ExecutePattern::Times(1))) => { let parsed_reg = format!("{reg}"); @@ -310,12 +319,19 @@ mod tests { _ => panic!("expected SkipToMatch pattern"), }; match patterns.get(4) { - Some(Pattern::SkipToMatch(reg, -3, ExecutePattern::Times(1))) => { + Some(Pattern::SkipToMatch(reg, 3, ExecutePattern::Times(1))) => { let parsed_reg = format!("{reg}"); assert_eq!(parsed_reg, "test5.*end$"); } _ => panic!("expected SkipToMatch pattern"), }; + match patterns.get(5) { + Some(Pattern::SkipToMatch(reg, -3, ExecutePattern::Times(1))) => { + let parsed_reg = format!("{reg}"); + assert_eq!(parsed_reg, "test6.*end$"); + } + _ => panic!("expected SkipToMatch pattern"), + }; } #[test] diff --git a/src/uu/csplit/src/split_name.rs b/src/uu/csplit/src/split_name.rs index 4d94b56a923..925ded4cc7b 100644 --- a/src/uu/csplit/src/split_name.rs +++ b/src/uu/csplit/src/split_name.rs @@ -4,21 +4,22 @@ // file that was distributed with this source code. // spell-checker:ignore (regex) diuox -use regex::Regex; +use uucore::format::{Format, FormatError, num_format::UnsignedInt}; use crate::csplit_error::CsplitError; /// Computes the filename of a split, taking into consideration a possible user-defined suffix /// format. pub struct SplitName { - fn_split_name: Box String>, + prefix: Vec, + format: Format, } impl SplitName { /// Creates a new SplitName with the given user-defined options: /// - `prefix_opt` specifies a prefix for all splits. /// - `format_opt` specifies a custom format for the suffix part of the filename, using the - /// `sprintf` format notation. + /// `sprintf` format notation. /// - `n_digits_opt` defines the width of the split number. /// /// # Caveats @@ -36,6 +37,7 @@ impl SplitName { ) -> Result { // get the prefix let prefix = prefix_opt.unwrap_or_else(|| "xx".to_string()); + // the width for the split offset let n_digits = n_digits_opt .map(|opt| { @@ -44,120 +46,26 @@ impl SplitName { }) .transpose()? .unwrap_or(2); - // translate the custom format into a function - let fn_split_name: Box String> = match format_opt { - None => Box::new(move |n: usize| -> String { format!("{prefix}{n:0n_digits$}") }), - Some(custom) => { - let spec = - Regex::new(r"(?P%((?P[0#-])(?P\d+)?)?(?P[diuoxX]))") - .unwrap(); - let mut captures_iter = spec.captures_iter(&custom); - let custom_fn: Box String> = match captures_iter.next() { - Some(captures) => { - let all = captures.name("ALL").unwrap(); - let before = custom[0..all.start()].to_owned(); - let after = custom[all.end()..].to_owned(); - let width = match captures.name("WIDTH") { - None => 0, - Some(m) => m.as_str().parse::().unwrap(), - }; - match (captures.name("FLAG"), captures.name("TYPE")) { - (None, Some(ref t)) => match t.as_str() { - "d" | "i" | "u" => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n}{after}") - }), - "o" => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:o}{after}") - }), - "x" => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:x}{after}") - }), - "X" => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:X}{after}") - }), - _ => return Err(CsplitError::SuffixFormatIncorrect), - }, - (Some(ref f), Some(ref t)) => { - match (f.as_str(), t.as_str()) { - /* - * zero padding - */ - // decimal - ("0", "d" | "i" | "u") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:0width$}{after}") - }), - // octal - ("0", "o") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:0width$o}{after}") - }), - // lower hexadecimal - ("0", "x") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:0width$x}{after}") - }), - // upper hexadecimal - ("0", "X") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:0width$X}{after}") - }), - - /* - * Alternate form - */ - // octal - ("#", "o") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:>#width$o}{after}") - }), - // lower hexadecimal - ("#", "x") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:>#width$x}{after}") - }), - // upper hexadecimal - ("#", "X") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:>#width$X}{after}") - }), - - /* - * Left adjusted - */ - // decimal - ("-", "d" | "i" | "u") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:<#width$}{after}") - }), - // octal - ("-", "o") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:<#width$o}{after}") - }), - // lower hexadecimal - ("-", "x") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:<#width$x}{after}") - }), - // upper hexadecimal - ("-", "X") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:<#width$X}{after}") - }), - - _ => return Err(CsplitError::SuffixFormatIncorrect), - } - } - _ => return Err(CsplitError::SuffixFormatIncorrect), - } - } - None => return Err(CsplitError::SuffixFormatIncorrect), - }; - - // there cannot be more than one format pattern - if captures_iter.next().is_some() { - return Err(CsplitError::SuffixFormatTooManyPercents); - } - custom_fn - } - }; - Ok(Self { fn_split_name }) + let format_string = format_opt.unwrap_or_else(|| format!("%0{n_digits}u")); + + let format = match Format::::parse(format_string) { + Ok(format) => Ok(format), + Err(FormatError::TooManySpecs(_)) => Err(CsplitError::SuffixFormatTooManyPercents), + Err(_) => Err(CsplitError::SuffixFormatIncorrect), + }?; + + Ok(Self { + prefix: prefix.as_bytes().to_owned(), + format, + }) } /// Returns the filename of the i-th split. pub fn get(&self, n: usize) -> String { - (self.fn_split_name)(n) + let mut v = self.prefix.clone(); + self.format.fmt(&mut v, n as u64).unwrap(); + String::from_utf8_lossy(&v).to_string() } } @@ -279,7 +187,7 @@ mod tests { #[test] fn alternate_form_octal() { let split_name = SplitName::new(None, Some(String::from("cst-%#10o-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst- 0o52-"); + assert_eq!(split_name.get(42), "xxcst- 052-"); } #[test] @@ -291,7 +199,7 @@ mod tests { #[test] fn alternate_form_upper_hex() { let split_name = SplitName::new(None, Some(String::from("cst-%#10X-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst- 0x2A-"); + assert_eq!(split_name.get(42), "xxcst- 0X2A-"); } #[test] @@ -315,19 +223,19 @@ mod tests { #[test] fn left_adjusted_octal() { let split_name = SplitName::new(None, Some(String::from("cst-%-10o-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst-0o52 -"); + assert_eq!(split_name.get(42), "xxcst-52 -"); } #[test] fn left_adjusted_lower_hex() { let split_name = SplitName::new(None, Some(String::from("cst-%-10x-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst-0x2a -"); + assert_eq!(split_name.get(42), "xxcst-2a -"); } #[test] fn left_adjusted_upper_hex() { let split_name = SplitName::new(None, Some(String::from("cst-%-10X-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst-0x2A -"); + assert_eq!(split_name.get(42), "xxcst-2A -"); } #[test] diff --git a/src/uu/cut/Cargo.toml b/src/uu/cut/Cargo.toml index c98bec5bc42..84fe09f23cf 100644 --- a/src/uu/cut/Cargo.toml +++ b/src/uu/cut/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_cut" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "cut ~ (uutils) display byte/field columns of input lines" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/cut" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/cut.rs" diff --git a/src/uu/cut/src/cut.rs b/src/uu/cut/src/cut.rs index 2a3196d002e..49f5445f36f 100644 --- a/src/uu/cut/src/cut.rs +++ b/src/uu/cut/src/cut.rs @@ -6,13 +6,15 @@ // spell-checker:ignore (ToDO) delim sourcefiles use bstr::io::BufReadExt; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command, builder::ValueParser}; +use std::ffi::OsString; use std::fs::File; -use std::io::{stdin, stdout, BufReader, BufWriter, IsTerminal, Read, Write}; +use std::io::{BufRead, BufReader, BufWriter, IsTerminal, Read, Write, stdin, stdout}; use std::path::Path; use uucore::display::Quotable; -use uucore::error::{FromIo, UResult, USimpleError}; +use uucore::error::{FromIo, UResult, USimpleError, set_exit_code}; use uucore::line_ending::LineEnding; +use uucore::os_str_as_bytes; use self::searcher::Searcher; use matcher::{ExactMatcher, Matcher, WhitespaceMatcher}; @@ -26,34 +28,37 @@ const USAGE: &str = help_usage!("cut.md"); const ABOUT: &str = help_about!("cut.md"); const AFTER_HELP: &str = help_section!("after help", "cut.md"); -struct Options { - out_delim: Option, +struct Options<'a> { + out_delimiter: Option<&'a [u8]>, line_ending: LineEnding, + field_opts: Option>, } -enum Delimiter { +enum Delimiter<'a> { Whitespace, - String(String), // FIXME: use char? + Slice(&'a [u8]), } -struct FieldOptions { - delimiter: Delimiter, - out_delimiter: Option, +struct FieldOptions<'a> { + delimiter: Delimiter<'a>, only_delimited: bool, - line_ending: LineEnding, } -enum Mode { - Bytes(Vec, Options), - Characters(Vec, Options), - Fields(Vec, FieldOptions), +enum Mode<'a> { + Bytes(Vec, Options<'a>), + Characters(Vec, Options<'a>), + Fields(Vec, Options<'a>), } -fn stdout_writer() -> Box { - if std::io::stdout().is_terminal() { - Box::new(stdout()) - } else { - Box::new(BufWriter::new(stdout())) as Box +impl Default for Delimiter<'_> { + fn default() -> Self { + Self::Slice(b"\t") + } +} + +impl<'a> From<&'a OsString> for Delimiter<'a> { + fn from(s: &'a OsString) -> Self { + Self::Slice(os_str_as_bytes(s).unwrap()) } } @@ -65,15 +70,15 @@ fn list_to_ranges(list: &str, complement: bool) -> Result, String> { } } -fn cut_bytes(reader: R, ranges: &[Range], opts: &Options) -> UResult<()> { +fn cut_bytes( + reader: R, + out: &mut W, + ranges: &[Range], + opts: &Options, +) -> UResult<()> { let newline_char = opts.line_ending.into(); let mut buf_in = BufReader::new(reader); - let mut out = stdout_writer(); - let delim = opts - .out_delim - .as_ref() - .map_or("", String::as_str) - .as_bytes(); + let out_delim = opts.out_delimiter.unwrap_or(b"\t"); let result = buf_in.for_byte_record(newline_char, |line| { let mut print_delim = false; @@ -82,8 +87,8 @@ fn cut_bytes(reader: R, ranges: &[Range], opts: &Options) -> UResult<() break; } if print_delim { - out.write_all(delim)?; - } else if opts.out_delim.is_some() { + out.write_all(out_delim)?; + } else if opts.out_delimiter.is_some() { print_delim = true; } // change `low` from 1-indexed value to 0-index value @@ -103,16 +108,16 @@ fn cut_bytes(reader: R, ranges: &[Range], opts: &Options) -> UResult<() } // Output delimiter is explicitly specified -fn cut_fields_explicit_out_delim( +fn cut_fields_explicit_out_delim( reader: R, + out: &mut W, matcher: &M, ranges: &[Range], only_delimited: bool, newline_char: u8, - out_delim: &str, + out_delim: &[u8], ) -> UResult<()> { let mut buf_in = BufReader::new(reader); - let mut out = stdout_writer(); let result = buf_in.for_byte_record_with_terminator(newline_char, |line| { let mut fields_pos = 1; @@ -122,8 +127,9 @@ fn cut_fields_explicit_out_delim( if delim_search.peek().is_none() { if !only_delimited { + // Always write the entire line, even if it doesn't end with `newline_char` out.write_all(line)?; - if line[line.len() - 1] != newline_char { + if line.is_empty() || line[line.len() - 1] != newline_char { out.write_all(&[newline_char])?; } } @@ -145,7 +151,7 @@ fn cut_fields_explicit_out_delim( for _ in 0..=high - low { // skip printing delimiter if this is the first matching field for this line if print_delim { - out.write_all(out_delim.as_bytes())?; + out.write_all(out_delim)?; } else { print_delim = true; } @@ -187,15 +193,15 @@ fn cut_fields_explicit_out_delim( } // Output delimiter is the same as input delimiter -fn cut_fields_implicit_out_delim( +fn cut_fields_implicit_out_delim( reader: R, + out: &mut W, matcher: &M, ranges: &[Range], only_delimited: bool, newline_char: u8, ) -> UResult<()> { let mut buf_in = BufReader::new(reader); - let mut out = stdout_writer(); let result = buf_in.for_byte_record_with_terminator(newline_char, |line| { let mut fields_pos = 1; @@ -205,8 +211,9 @@ fn cut_fields_implicit_out_delim( if delim_search.peek().is_none() { if !only_delimited { + // Always write the entire line, even if it doesn't end with `newline_char` out.write_all(line)?; - if line[line.len() - 1] != newline_char { + if line.is_empty() || line[line.len() - 1] != newline_char { out.write_all(&[newline_char])?; } } @@ -256,39 +263,83 @@ fn cut_fields_implicit_out_delim( Ok(()) } -fn cut_fields(reader: R, ranges: &[Range], opts: &FieldOptions) -> UResult<()> { +// The input delimiter is identical to `newline_char` +fn cut_fields_newline_char_delim( + reader: R, + out: &mut W, + ranges: &[Range], + newline_char: u8, + out_delim: &[u8], +) -> UResult<()> { + let buf_in = BufReader::new(reader); + + let segments: Vec<_> = buf_in.split(newline_char).filter_map(|x| x.ok()).collect(); + let mut print_delim = false; + + for &Range { low, high } in ranges { + for i in low..=high { + // "- 1" is necessary because fields start from 1 whereas a Vec starts from 0 + if let Some(segment) = segments.get(i - 1) { + if print_delim { + out.write_all(out_delim)?; + } else { + print_delim = true; + } + out.write_all(segment.as_slice())?; + } else { + break; + } + } + } + out.write_all(&[newline_char])?; + Ok(()) +} + +fn cut_fields( + reader: R, + out: &mut W, + ranges: &[Range], + opts: &Options, +) -> UResult<()> { let newline_char = opts.line_ending.into(); - match opts.delimiter { - Delimiter::String(ref delim) => { - let matcher = ExactMatcher::new(delim.as_bytes()); + let field_opts = opts.field_opts.as_ref().unwrap(); // it is safe to unwrap() here - field_opts will always be Some() for cut_fields() call + match field_opts.delimiter { + Delimiter::Slice(delim) if delim == [newline_char] => { + let out_delim = opts.out_delimiter.unwrap_or(delim); + cut_fields_newline_char_delim(reader, out, ranges, newline_char, out_delim) + } + Delimiter::Slice(delim) => { + let matcher = ExactMatcher::new(delim); match opts.out_delimiter { - Some(ref out_delim) => cut_fields_explicit_out_delim( + Some(out_delim) => cut_fields_explicit_out_delim( reader, + out, &matcher, ranges, - opts.only_delimited, + field_opts.only_delimited, newline_char, out_delim, ), None => cut_fields_implicit_out_delim( reader, + out, &matcher, ranges, - opts.only_delimited, + field_opts.only_delimited, newline_char, ), } } Delimiter::Whitespace => { let matcher = WhitespaceMatcher {}; - let out_delim = opts.out_delimiter.as_deref().unwrap_or("\t"); cut_fields_explicit_out_delim( reader, + out, &matcher, ranges, - opts.only_delimited, + field_opts.only_delimited, newline_char, - out_delim, + opts.out_delimiter.unwrap_or(b"\t"), ) } } @@ -301,6 +352,12 @@ fn cut_files(mut filenames: Vec, mode: &Mode) { filenames.push("-".to_owned()); } + let mut out: Box = if stdout().is_terminal() { + Box::new(stdout()) + } else { + Box::new(BufWriter::new(stdout())) as Box + }; + for filename in &filenames { if filename == "-" { if stdin_read { @@ -308,9 +365,9 @@ fn cut_files(mut filenames: Vec, mode: &Mode) { } show_if_err!(match mode { - Mode::Bytes(ref ranges, ref opts) => cut_bytes(stdin(), ranges, opts), - Mode::Characters(ref ranges, ref opts) => cut_bytes(stdin(), ranges, opts), - Mode::Fields(ref ranges, ref opts) => cut_fields(stdin(), ranges, opts), + Mode::Bytes(ranges, opts) => cut_bytes(stdin(), &mut out, ranges, opts), + Mode::Characters(ranges, opts) => cut_bytes(stdin(), &mut out, ranges, opts), + Mode::Fields(ranges, opts) => cut_fields(stdin(), &mut out, ranges, opts), }); stdin_read = true; @@ -319,21 +376,78 @@ fn cut_files(mut filenames: Vec, mode: &Mode) { if path.is_dir() { show_error!("{}: Is a directory", filename.maybe_quote()); + set_exit_code(1); continue; } - show_if_err!(File::open(path) - .map_err_context(|| filename.maybe_quote().to_string()) - .and_then(|file| { - match &mode { - Mode::Bytes(ranges, opts) | Mode::Characters(ranges, opts) => { - cut_bytes(file, ranges, opts) + show_if_err!( + File::open(path) + .map_err_context(|| filename.maybe_quote().to_string()) + .and_then(|file| { + match &mode { + Mode::Bytes(ranges, opts) | Mode::Characters(ranges, opts) => { + cut_bytes(file, &mut out, ranges, opts) + } + Mode::Fields(ranges, opts) => cut_fields(file, &mut out, ranges, opts), } - Mode::Fields(ranges, opts) => cut_fields(file, ranges, opts), - } - })); + }) + ); } } + + show_if_err!(out.flush().map_err_context(|| "write error".into())); +} + +// Get delimiter and output delimiter from `-d`/`--delimiter` and `--output-delimiter` options respectively +// Allow either delimiter to have a value that is neither UTF-8 nor ASCII to align with GNU behavior +fn get_delimiters(matches: &ArgMatches) -> UResult<(Delimiter, Option<&[u8]>)> { + let whitespace_delimited = matches.get_flag(options::WHITESPACE_DELIMITED); + let delim_opt = matches.get_one::(options::DELIMITER); + let delim = match delim_opt { + Some(_) if whitespace_delimited => { + return Err(USimpleError::new( + 1, + "invalid input: Only one of --delimiter (-d) or -w option can be specified", + )); + } + Some(os_string) => { + if os_string == "''" || os_string.is_empty() { + // treat `''` as empty delimiter + Delimiter::Slice(b"\0") + } else { + // For delimiter `-d` option value - allow both UTF-8 (possibly multi-byte) characters + // and Non UTF-8 (and not ASCII) single byte "characters", like `b"\xAD"` to align with GNU behavior + let bytes = os_str_as_bytes(os_string)?; + if os_string.to_str().is_some_and(|s| s.chars().count() > 1) + || os_string.to_str().is_none() && bytes.len() > 1 + { + return Err(USimpleError::new( + 1, + "the delimiter must be a single character", + )); + } else { + Delimiter::from(os_string) + } + } + } + None => { + if whitespace_delimited { + Delimiter::Whitespace + } else { + Delimiter::default() + } + } + }; + let out_delim = matches + .get_one::(options::OUTPUT_DELIMITER) + .map(|os_string| { + if os_string.is_empty() || os_string == "''" { + b"\0" + } else { + os_str_as_bytes(os_string).unwrap() + } + }); + Ok((delim, out_delim)) } mod options { @@ -351,116 +465,76 @@ mod options { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_ignore(); + // GNU's `cut` supports `-d=` to set the delimiter to `=`. + // Clap parsing is limited in this situation, see: + // https://github.com/uutils/coreutils/issues/2424#issuecomment-863825242 + let args: Vec = args + .into_iter() + .map(|x| { + if x == "-d=" { + "--delimiter==".into() + } else { + x + } + }) + .collect(); - let delimiter_is_equal = args.contains(&"-d=".to_string()); // special case let matches = uu_app().try_get_matches_from(args)?; let complement = matches.get_flag(options::COMPLEMENT); + let only_delimited = matches.get_flag(options::ONLY_DELIMITED); + + let (delimiter, out_delimiter) = get_delimiters(&matches)?; + let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO_TERMINATED)); + + // Only one, and only one of cutting mode arguments, i.e. `-b`, `-c`, `-f`, + // is expected. The number of those arguments is used for parsing a cutting + // mode and handling the error cases. + let mode_args_count = [ + matches.indices_of(options::BYTES), + matches.indices_of(options::CHARACTERS), + matches.indices_of(options::FIELDS), + ] + .into_iter() + .map(|indices| indices.unwrap_or_default().count()) + .sum(); let mode_parse = match ( + mode_args_count, matches.get_one::(options::BYTES), matches.get_one::(options::CHARACTERS), matches.get_one::(options::FIELDS), ) { - (Some(byte_ranges), None, None) => list_to_ranges(byte_ranges, complement).map(|ranges| { + (1, Some(byte_ranges), None, None) => list_to_ranges(byte_ranges, complement).map(|ranges| { Mode::Bytes( ranges, Options { - out_delim: Some( - matches - .get_one::(options::OUTPUT_DELIMITER) - .map(|s| s.as_str()) - .unwrap_or_default() - .to_owned(), - ), - line_ending: LineEnding::from_zero_flag(matches.get_flag(options::ZERO_TERMINATED)), + out_delimiter, + line_ending, + field_opts: None, }, ) }), - (None, Some(char_ranges), None) => list_to_ranges(char_ranges, complement).map(|ranges| { + (1, None, Some(char_ranges), None) => list_to_ranges(char_ranges, complement).map(|ranges| { Mode::Characters( ranges, Options { - out_delim: Some( - matches - .get_one::(options::OUTPUT_DELIMITER) - .map(|s| s.as_str()) - .unwrap_or_default() - .to_owned(), - ), - line_ending: LineEnding::from_zero_flag(matches.get_flag(options::ZERO_TERMINATED)), + out_delimiter, + line_ending, + field_opts: None, }, ) }), - (None, None, Some(field_ranges)) => { - list_to_ranges(field_ranges, complement).and_then(|ranges| { - let out_delim = match matches.get_one::(options::OUTPUT_DELIMITER) { - Some(s) => { - if s.is_empty() { - Some("\0".to_owned()) - } else { - Some(s.clone()) - } - } - None => None, - }; - - let only_delimited = matches.get_flag(options::ONLY_DELIMITED); - let whitespace_delimited = matches.get_flag(options::WHITESPACE_DELIMITED); - let zero_terminated = matches.get_flag(options::ZERO_TERMINATED); - let line_ending = LineEnding::from_zero_flag(zero_terminated); - - match matches.get_one::(options::DELIMITER).map(|s| s.as_str()) { - Some(_) if whitespace_delimited => { - Err("invalid input: Only one of --delimiter (-d) or -w option can be specified".into()) - } - Some(mut delim) => { - // GNU's `cut` supports `-d=` to set the delimiter to `=`. - // Clap parsing is limited in this situation, see: - // https://github.com/uutils/coreutils/issues/2424#issuecomment-863825242 - if delimiter_is_equal { - delim = "="; - } else if delim == "''" { - // treat `''` as empty delimiter - delim = ""; - } - if delim.chars().count() > 1 { - Err("the delimiter must be a single character".into()) - } else { - let delim = if delim.is_empty() { - "\0".to_owned() - } else { - delim.to_owned() - }; - - Ok(Mode::Fields( - ranges, - FieldOptions { - delimiter: Delimiter::String(delim), - out_delimiter: out_delim, - only_delimited, - line_ending, - }, - )) - } - } - None => Ok(Mode::Fields( - ranges, - FieldOptions { - delimiter: match whitespace_delimited { - true => Delimiter::Whitespace, - false => Delimiter::String("\t".to_owned()), - }, - out_delimiter: out_delim, - only_delimited, - line_ending, - }, - )), - } - }) - } - (ref b, ref c, ref f) if b.is_some() || c.is_some() || f.is_some() => Err( + (1, None, None, Some(field_ranges)) => list_to_ranges(field_ranges, complement).map(|ranges| { + Mode::Fields( + ranges, + Options { + out_delimiter, + line_ending, + field_opts: Some(FieldOptions { delimiter, only_delimited })}, + ) + }), + (2.., _, _, _) => Err( "invalid usage: expects no more than one of --fields (-f), --chars (-c) or --bytes (-b)".into() ), _ => Err("invalid usage: expects one of --fields (-f), --chars (-c) or --bytes (-b)".into()), @@ -505,11 +579,18 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .override_usage(format_usage(USAGE)) .about(ABOUT) .after_help(AFTER_HELP) .infer_long_args(true) + // While `args_override_self(true)` for some arguments, such as `-d` + // and `--output-delimiter`, is consistent to the behavior of GNU cut, + // arguments related to cutting mode, i.e. `-b`, `-c`, `-f`, should + // cause an error when there is more than one of them, as described in + // the manual of GNU cut: "Use one, and only one of -b, -c or -f". + // `ArgAction::Append` is used on `-b`, `-c`, `-f` arguments, so that + // the occurrences of those could be counted and be handled accordingly. .args_override_self(true) .arg( Arg::new(options::BYTES) @@ -517,7 +598,8 @@ pub fn uu_app() -> Command { .long(options::BYTES) .help("filter byte columns from the input source") .allow_hyphen_values(true) - .value_name("LIST"), + .value_name("LIST") + .action(ArgAction::Append), ) .arg( Arg::new(options::CHARACTERS) @@ -525,12 +607,14 @@ pub fn uu_app() -> Command { .long(options::CHARACTERS) .help("alias for character mode") .allow_hyphen_values(true) - .value_name("LIST"), + .value_name("LIST") + .action(ArgAction::Append), ) .arg( Arg::new(options::DELIMITER) .short('d') .long(options::DELIMITER) + .value_parser(ValueParser::os_string()) .help("specify the delimiter character that separates fields in the input source. Defaults to Tab.") .value_name("DELIM"), ) @@ -547,7 +631,8 @@ pub fn uu_app() -> Command { .long(options::FIELDS) .help("filter field columns from the input source") .allow_hyphen_values(true) - .value_name("LIST"), + .value_name("LIST") + .action(ArgAction::Append), ) .arg( Arg::new(options::COMPLEMENT) @@ -572,6 +657,7 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::OUTPUT_DELIMITER) .long(options::OUTPUT_DELIMITER) + .value_parser(ValueParser::os_string()) .help("in field mode, replace the delimiter in output lines with this option's argument") .value_name("NEW_DELIM"), ) diff --git a/src/uu/cut/src/matcher.rs b/src/uu/cut/src/matcher.rs index 953e083b139..bb0c44d5bb4 100644 --- a/src/uu/cut/src/matcher.rs +++ b/src/uu/cut/src/matcher.rs @@ -23,7 +23,7 @@ impl<'a> ExactMatcher<'a> { } } -impl<'a> Matcher for ExactMatcher<'a> { +impl Matcher for ExactMatcher<'_> { fn next_match(&self, haystack: &[u8]) -> Option<(usize, usize)> { let mut pos = 0usize; loop { diff --git a/src/uu/cut/src/searcher.rs b/src/uu/cut/src/searcher.rs index 21424790eea..dc252d804f7 100644 --- a/src/uu/cut/src/searcher.rs +++ b/src/uu/cut/src/searcher.rs @@ -27,7 +27,7 @@ impl<'a, 'b, M: Matcher> Searcher<'a, 'b, M> { // Iterate over field delimiters // Returns (first, last) positions of each sequence, where `haystack[first..last]` // corresponds to the delimiter. -impl<'a, 'b, M: Matcher> Iterator for Searcher<'a, 'b, M> { +impl Iterator for Searcher<'_, '_, M> { type Item = (usize, usize); fn next(&mut self) -> Option { @@ -61,7 +61,7 @@ mod exact_searcher_tests { let matcher = ExactMatcher::new("a".as_bytes()); let iter = Searcher::new(&matcher, "".as_bytes()); let items: Vec<(usize, usize)> = iter.collect(); - assert_eq!(vec![] as Vec<(usize, usize)>, items); + assert!(items.is_empty()); } fn test_multibyte(line: &[u8], expected: &[(usize, usize)]) { @@ -140,7 +140,7 @@ mod whitespace_searcher_tests { let matcher = WhitespaceMatcher {}; let iter = Searcher::new(&matcher, "".as_bytes()); let items: Vec<(usize, usize)> = iter.collect(); - assert_eq!(vec![] as Vec<(usize, usize)>, items); + assert!(items.is_empty()); } fn test_multispace(line: &[u8], expected: &[(usize, usize)]) { diff --git a/src/uu/date/Cargo.toml b/src/uu/date/Cargo.toml index 11ad64bbef7..087d4befc7e 100644 --- a/src/uu/date/Cargo.toml +++ b/src/uu/date/Cargo.toml @@ -1,16 +1,19 @@ # spell-checker:ignore datetime [package] name = "uu_date" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "date ~ (uutils) display or set the current time" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/date" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/date.rs" @@ -18,7 +21,7 @@ path = "src/date.rs" [dependencies] chrono = { workspace = true } clap = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["custom-tz-fmt", "parser"] } parse_datetime = { workspace = true } [target.'cfg(unix)'.dependencies] diff --git a/src/uu/date/date-usage.md b/src/uu/date/date-usage.md index bf2dc469d3a..109bfd3988b 100644 --- a/src/uu/date/date-usage.md +++ b/src/uu/date/date-usage.md @@ -79,9 +79,3 @@ Show the time on the west coast of the US (use tzselect(1) to find TZ) ``` TZ='America/Los_Angeles' date ``` - -Show the local time for 9AM next Friday on the west coast of the US - -``` -date --date='TZ="America/Los_Angeles" 09:00 next Fri' -``` diff --git a/src/uu/date/src/date.rs b/src/uu/date/src/date.rs index ee3c7bfdfae..f4c9313cb62 100644 --- a/src/uu/date/src/date.rs +++ b/src/uu/date/src/date.rs @@ -6,15 +6,16 @@ // spell-checker:ignore (chrono) Datelike Timelike ; (format) DATEFILE MMDDhhmm ; (vars) datetime datetimes use chrono::format::{Item, StrftimeItems}; -use chrono::{DateTime, Duration, FixedOffset, Local, Offset, Utc}; +use chrono::{DateTime, FixedOffset, Local, Offset, TimeDelta, Utc}; #[cfg(windows)] use chrono::{Datelike, Timelike}; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; #[cfg(all(unix, not(target_os = "macos"), not(target_os = "redox")))] -use libc::{clock_settime, timespec, CLOCK_REALTIME}; +use libc::{CLOCK_REALTIME, clock_settime, timespec}; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::PathBuf; +use uucore::custom_tz_fmt::custom_time_format; use uucore::display::Quotable; use uucore::error::FromIo; use uucore::error::{UResult, USimpleError}; @@ -22,7 +23,7 @@ use uucore::{format_usage, help_about, help_usage, show}; #[cfg(windows)] use windows_sys::Win32::{Foundation::SYSTEMTIME, System::SystemInformation::SetSystemTime}; -use uucore::shortcut_value_parser::ShortcutValueParser; +use uucore::parser::shortcut_value_parser::ShortcutValueParser; // Options const DATE: &str = "date"; @@ -91,7 +92,8 @@ enum DateSource { Now, Custom(String), File(PathBuf), - Human(Duration), + Stdin, + Human(TimeDelta), } enum Iso8601Format { @@ -102,7 +104,7 @@ enum Iso8601Format { Ns, } -impl<'a> From<&'a str> for Iso8601Format { +impl From<&str> for Iso8601Format { fn from(s: &str) -> Self { match s { HOURS => Self::Hours, @@ -122,7 +124,7 @@ enum Rfc3339Format { Ns, } -impl<'a> From<&'a str> for Rfc3339Format { +impl From<&str> for Rfc3339Format { fn from(s: &str) -> Self { match s { DATE => Self::Date, @@ -173,7 +175,10 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { DateSource::Custom(date.into()) } } else if let Some(file) = matches.get_one::(OPT_FILE) { - DateSource::File(file.into()) + match file.as_ref() { + "-" => DateSource::Stdin, + _ => DateSource::File(file.into()), + } } else { DateSource::Now }; @@ -223,11 +228,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { Box::new(iter) } DateSource::Human(relative_time) => { - // Get the current DateTime for things like "1 year ago" - let current_time = DateTime::::from(Local::now()); - // double check the result is overflow or not of the current_time + relative_time + // Double check the result is overflow or not of the current_time + relative_time // it may cause a panic of chrono::datetime::DateTime add - match current_time.checked_add_signed(relative_time) { + match now.checked_add_signed(relative_time) { Some(date) => { let iter = std::iter::once(Ok(date)); Box::new(iter) @@ -235,11 +238,16 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { None => { return Err(USimpleError::new( 1, - format!("invalid date {}", relative_time), + format!("invalid date {relative_time}"), )); } } } + DateSource::Stdin => { + let lines = BufReader::new(std::io::stdin()).lines(); + let iter = lines.map_while(Result::ok).map(parse_date); + Box::new(iter) + } DateSource::File(ref path) => { if path.is_dir() { return Err(USimpleError::new( @@ -265,18 +273,10 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { for date in dates { match date { Ok(date) => { - // GNU `date` uses `%N` for nano seconds, however crate::chrono uses `%f` - let format_string = &format_string.replace("%N", "%f"); - // Refuse to pass this string to chrono as it is crashing in this crate - if format_string.contains("%#z") { - return Err(USimpleError::new( - 1, - format!("invalid format {}", format_string.replace("%f", "%N")), - )); - } + let format_string = custom_time_format(format_string); // Hack to work around panic in chrono, // TODO - remove when a fix for https://github.com/chronotope/chrono/issues/623 is released - let format_items = StrftimeItems::new(format_string); + let format_items = StrftimeItems::new(format_string.as_str()); if format_items.clone().any(|i| i == Item::Error) { return Err(USimpleError::new( 1, @@ -302,7 +302,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -311,6 +311,7 @@ pub fn uu_app() -> Command { .short('d') .long(OPT_DATE) .value_name("STRING") + .allow_hyphen_values(true) .help("display time described by STRING, not 'now'"), ) .arg( @@ -396,7 +397,7 @@ fn make_format_string(settings: &Settings) -> &str { Rfc3339Format::Ns => "%F %T.%f%:z", }, Format::Custom(ref fmt) => fmt, - Format::Default => "%c", + Format::Default => "%a %b %e %X %Z %Y", } } @@ -404,9 +405,8 @@ fn make_format_string(settings: &Settings) -> &str { /// If it fails, return a tuple of the `String` along with its `ParseError`. fn parse_date + Clone>( s: S, -) -> Result, (String, chrono::format::ParseError)> { - // TODO: The GNU date command can parse a wide variety of inputs. - s.as_ref().parse().map_err(|e| (s.as_ref().into(), e)) +) -> Result, (String, parse_datetime::ParseDateTimeError)> { + parse_datetime::parse_datetime(s.as_ref()).map_err(|e| (s.as_ref().into(), e)) } #[cfg(not(any(unix, windows)))] diff --git a/src/uu/dd/Cargo.toml b/src/uu/dd/Cargo.toml index 1dbb37bde55..04f05179926 100644 --- a/src/uu/dd/Cargo.toml +++ b/src/uu/dd/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_dd" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "dd ~ (uutils) copy and convert files" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/dd" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/dd.rs" @@ -18,13 +21,17 @@ path = "src/dd.rs" clap = { workspace = true } gcd = { workspace = true } libc = { workspace = true } -uucore = { workspace = true, features = ["format", "quoting-style"] } - -[target.'cfg(any(target_os = "linux"))'.dependencies] -nix = { workspace = true, features = ["fs"] } +uucore = { workspace = true, features = [ + "format", + "parser", + "quoting-style", + "fs", +] } +thiserror = { workspace = true } [target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies] signal-hook = { workspace = true } +nix = { workspace = true, features = ["fs"] } [[bin]] name = "dd" diff --git a/src/uu/dd/src/blocks.rs b/src/uu/dd/src/blocks.rs index 8e5557a2c9b..b7449c98be7 100644 --- a/src/uu/dd/src/blocks.rs +++ b/src/uu/dd/src/blocks.rs @@ -133,7 +133,7 @@ mod tests { let buf = [0u8, 1u8, 2u8, 3u8]; let res = block(&buf, 4, false, &mut rs); - assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8],]); + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8]]); } #[test] @@ -144,7 +144,7 @@ mod tests { assert_eq!( res, - vec![vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE],] + vec![vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE]] ); } @@ -155,7 +155,7 @@ mod tests { let res = block(&buf, 4, false, &mut rs); // Commented section(s) should be truncated and appear for reference only. - assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8 /*, 4u8*/],]); + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8 /*, 4u8*/]]); assert_eq!(rs.records_truncated, 1); } @@ -238,7 +238,7 @@ mod tests { let buf = [0u8, 1u8, 2u8, 3u8, NEWLINE]; let res = block(&buf, 4, false, &mut rs); - assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8],]); + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8]]); } #[test] @@ -258,7 +258,7 @@ mod tests { assert_eq!( res, - vec![vec![0u8, 1u8, 2u8, SPACE], vec![SPACE, SPACE, SPACE, SPACE],] + vec![vec![0u8, 1u8, 2u8, SPACE], vec![SPACE, SPACE, SPACE, SPACE]] ); } @@ -270,7 +270,7 @@ mod tests { assert_eq!( res, - vec![vec![SPACE, SPACE, SPACE, SPACE], vec![0u8, 1u8, 2u8, 3u8],] + vec![vec![SPACE, SPACE, SPACE, SPACE], vec![0u8, 1u8, 2u8, 3u8]] ); } @@ -315,7 +315,7 @@ mod tests { let buf = [0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8]; let res = unblock(&buf, 8); - assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, NEWLINE],); + assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, NEWLINE]); } #[test] @@ -323,7 +323,7 @@ mod tests { let buf = [SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE]; let res = unblock(&buf, 8); - assert_eq!(res, vec![NEWLINE],); + assert_eq!(res, vec![NEWLINE]); } #[test] @@ -342,7 +342,7 @@ mod tests { let buf = [0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE]; let res = unblock(&buf, 8); - assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, NEWLINE],); + assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, NEWLINE]); } #[test] diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index 07a754deb51..0d79cdda4af 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore fname, ftype, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, iseek, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, oseek, outfile, parseargs, rlen, rmax, rremain, rsofar, rstat, sigusr, wlen, wstat seekable oconv canonicalized fadvise Fadvise FADV DONTNEED ESPIPE bufferedoutput +// spell-checker:ignore fname, ftype, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, iseek, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, oseek, outfile, parseargs, rlen, rmax, rremain, rsofar, rstat, sigusr, wlen, wstat seekable oconv canonicalized fadvise Fadvise FADV DONTNEED ESPIPE bufferedoutput, SETFL mod blocks; mod bufferedoutput; @@ -16,8 +16,14 @@ mod progress; use crate::bufferedoutput::BufferedOutput; use blocks::conv_block_unblock_helper; use datastructures::*; +#[cfg(any(target_os = "linux", target_os = "android"))] +use nix::fcntl::FcntlArg::F_SETFL; +#[cfg(any(target_os = "linux", target_os = "android"))] +use nix::fcntl::OFlag; use parseargs::Parser; -use progress::{gen_prog_updater, ProgUpdate, ReadStat, StatusLevel, WriteStat}; +use progress::ProgUpdateType; +use progress::{ProgUpdate, ReadStat, StatusLevel, WriteStat, gen_prog_updater}; +use uucore::io::OwnedFileDescriptorOrHandle; use std::cmp; use std::env; @@ -31,25 +37,25 @@ use std::os::unix::{ fs::FileTypeExt, io::{AsRawFd, FromRawFd}, }; +#[cfg(windows)] +use std::os::windows::{fs::MetadataExt, io::AsHandle}; use std::path::Path; -use std::sync::{ - atomic::{AtomicBool, Ordering::Relaxed}, - mpsc, Arc, -}; +use std::sync::atomic::AtomicU8; +use std::sync::{Arc, atomic::Ordering::Relaxed, mpsc}; use std::thread; use std::time::{Duration, Instant}; -use clap::{crate_version, Arg, Command}; +use clap::{Arg, Command}; use gcd::Gcd; #[cfg(target_os = "linux")] use nix::{ errno::Errno, - fcntl::{posix_fadvise, PosixFadviseAdvice}, + fcntl::{PosixFadviseAdvice, posix_fadvise}, }; use uucore::display::Quotable; -#[cfg(unix)] -use uucore::error::set_exit_code; use uucore::error::{FromIo, UResult}; +#[cfg(unix)] +use uucore::error::{USimpleError, set_exit_code}; #[cfg(target_os = "linux")] use uucore::show_if_err; use uucore::{format_usage, help_about, help_section, help_usage, show_error}; @@ -80,38 +86,65 @@ struct Settings { /// A timer which triggers on a given interval /// -/// After being constructed with [`Alarm::with_interval`], [`Alarm::is_triggered`] -/// will return true once per the given [`Duration`]. +/// After being constructed with [`Alarm::with_interval`], [`Alarm::get_trigger`] +/// will return [`ALARM_TRIGGER_TIMER`] once per the given [`Duration`]. +/// Alarm can be manually triggered with closure returned by [`Alarm::manual_trigger_fn`]. +/// [`Alarm::get_trigger`] will return [`ALARM_TRIGGER_SIGNAL`] in this case. /// /// Can be cloned, but the trigger status is shared across all instances so only /// the first caller each interval will yield true. /// /// When all instances are dropped the background thread will exit on the next interval. -#[derive(Debug, Clone)] pub struct Alarm { interval: Duration, - trigger: Arc, + trigger: Arc, } +pub const ALARM_TRIGGER_NONE: u8 = 0; +pub const ALARM_TRIGGER_TIMER: u8 = 1; +pub const ALARM_TRIGGER_SIGNAL: u8 = 2; + impl Alarm { + /// use to construct alarm timer with duration pub fn with_interval(interval: Duration) -> Self { - let trigger = Arc::new(AtomicBool::default()); + let trigger = Arc::new(AtomicU8::default()); let weak_trigger = Arc::downgrade(&trigger); thread::spawn(move || { while let Some(trigger) = weak_trigger.upgrade() { thread::sleep(interval); - trigger.store(true, Relaxed); + trigger.store(ALARM_TRIGGER_TIMER, Relaxed); } }); Self { interval, trigger } } - pub fn is_triggered(&self) -> bool { - self.trigger.swap(false, Relaxed) + /// Returns a closure that allows to manually trigger the alarm + /// + /// This is useful for cases where more than one alarm even source exists + /// In case of `dd` there is the SIGUSR1/SIGINFO case where we want to + /// trigger an manual progress report. + pub fn manual_trigger_fn(&self) -> Box { + let weak_trigger = Arc::downgrade(&self.trigger); + Box::new(move || { + if let Some(trigger) = weak_trigger.upgrade() { + trigger.store(ALARM_TRIGGER_SIGNAL, Relaxed); + } + }) + } + + /// Use this function to poll for any pending alarm event + /// + /// Returns `ALARM_TRIGGER_NONE` for no pending event. + /// Returns `ALARM_TRIGGER_TIMER` if the event was triggered by timer + /// Returns `ALARM_TRIGGER_SIGNAL` if the event was triggered manually + /// by the closure returned from `manual_trigger_fn` + pub fn get_trigger(&self) -> u8 { + self.trigger.swap(ALARM_TRIGGER_NONE, Relaxed) } + // Getter function for the configured interval duration pub fn get_interval(&self) -> Duration { self.interval } @@ -189,7 +222,8 @@ impl Source { /// The length of the data source in number of bytes. /// /// If it cannot be determined, then this function returns 0. - fn len(&self) -> std::io::Result { + fn len(&self) -> io::Result { + #[allow(clippy::match_wildcard_for_single_variants)] match self { Self::File(f) => Ok(f.metadata()?.len().try_into().unwrap_or(i64::MAX)), _ => Ok(0), @@ -227,7 +261,7 @@ impl Source { Err(e) => Err(e), } } - Self::File(f) => f.seek(io::SeekFrom::Start(n)), + Self::File(f) => f.seek(SeekFrom::Current(n.try_into().unwrap())), #[cfg(unix)] Self::Fifo(f) => io::copy(&mut f.take(n), &mut io::sink()), } @@ -241,6 +275,7 @@ impl Source { /// then this function returns an error. #[cfg(target_os = "linux")] fn discard_cache(&self, offset: libc::off_t, len: libc::off_t) -> nix::Result<()> { + #[allow(clippy::match_wildcard_for_single_variants)] match self { Self::File(f) => { let advice = PosixFadviseAdvice::POSIX_FADV_DONTNEED; @@ -283,9 +318,35 @@ impl<'a> Input<'a> { /// Instantiate this struct with stdin as a source. fn new_stdin(settings: &'a Settings) -> UResult { #[cfg(not(unix))] - let mut src = Source::Stdin(io::stdin()); + let mut src = { + let f = File::from(io::stdin().as_handle().try_clone_to_owned()?); + let is_file = if let Ok(metadata) = f.metadata() { + // this hack is needed as there is no other way on windows + // to differentiate between the case where `seek` works + // on a file handle or not. i.e. when the handle is no real + // file but a pipe, `seek` is still successful, but following + // `read`s are not affected by the seek. + metadata.creation_time() != 0 + } else { + false + }; + if is_file { + Source::File(f) + } else { + Source::Stdin(io::stdin()) + } + }; #[cfg(unix)] let mut src = Source::stdin_as_file(); + #[cfg(unix)] + if let Source::StdinFile(f) = &src { + if settings.iflags.directory && !f.metadata()?.is_dir() { + return Err(USimpleError::new( + 1, + "setting flags for 'standard input': Not a directory", + )); + } + }; if settings.skip > 0 { src.skip(settings.skip)?; } @@ -358,14 +419,10 @@ fn make_linux_iflags(iflags: &IFlags) -> Option { flag |= libc::O_SYNC; } - if flag == 0 { - None - } else { - Some(flag) - } + if flag == 0 { None } else { Some(flag) } } -impl<'a> Read for Input<'a> { +impl Read for Input<'_> { fn read(&mut self, buf: &mut [u8]) -> io::Result { let mut base_idx = 0; let target_len = buf.len(); @@ -388,7 +445,7 @@ impl<'a> Read for Input<'a> { } } -impl<'a> Input<'a> { +impl Input<'_> { /// Discard the system file cache for the given portion of the input. /// /// `offset` and `len` specify a contiguous portion of the input. @@ -396,7 +453,7 @@ impl<'a> Input<'a> { /// the input file is no longer needed. If not possible, then this /// function prints an error message to stderr and sets the exit /// status code to 1. - #[allow(unused_variables)] + #[cfg_attr(not(target_os = "linux"), allow(clippy::unused_self, unused_variables))] fn discard_cache(&self, offset: libc::off_t, len: libc::off_t) { #[cfg(target_os = "linux")] { @@ -415,7 +472,7 @@ impl<'a> Input<'a> { /// Fills a given buffer. /// Reads in increments of 'self.ibs'. /// The start of each ibs-sized read follows the previous one. - fn fill_consecutive(&mut self, buf: &mut Vec) -> std::io::Result { + fn fill_consecutive(&mut self, buf: &mut Vec) -> io::Result { let mut reads_complete = 0; let mut reads_partial = 0; let mut bytes_total = 0; @@ -446,7 +503,7 @@ impl<'a> Input<'a> { /// Fills a given buffer. /// Reads in increments of 'self.ibs'. /// The start of each ibs-sized read is aligned to multiples of ibs; remaining space is filled with the 'pad' byte. - fn fill_blocks(&mut self, buf: &mut Vec, pad: u8) -> std::io::Result { + fn fill_blocks(&mut self, buf: &mut Vec, pad: u8) -> io::Result { let mut reads_complete = 0; let mut reads_partial = 0; let mut base_idx = 0; @@ -557,7 +614,7 @@ impl Dest { return Ok(len); } } - f.seek(io::SeekFrom::Start(n)) + f.seek(SeekFrom::Current(n.try_into().unwrap())) } #[cfg(unix)] Self::Fifo(f) => { @@ -571,6 +628,7 @@ impl Dest { /// Truncate the underlying file to the current stream position, if possible. fn truncate(&mut self) -> io::Result<()> { + #[allow(clippy::match_wildcard_for_single_variants)] match self { Self::File(f, _) => { let pos = f.stream_position()?; @@ -600,7 +658,8 @@ impl Dest { /// The length of the data destination in number of bytes. /// /// If it cannot be determined, then this function returns 0. - fn len(&self) -> std::io::Result { + fn len(&self) -> io::Result { + #[allow(clippy::match_wildcard_for_single_variants)] match self { Self::File(f, _) => Ok(f.metadata()?.len().try_into().unwrap_or(i64::MAX)), _ => Ok(0), @@ -621,7 +680,7 @@ impl Write for Dest { .len() .try_into() .expect("Internal dd Error: Seek amount greater than signed 64-bit integer"); - f.seek(io::SeekFrom::Current(seek_amt))?; + f.seek(SeekFrom::Current(seek_amt))?; Ok(buf.len()) } Self::File(f, _) => f.write(buf), @@ -699,6 +758,11 @@ impl<'a> Output<'a> { if !settings.oconv.notrunc { dst.set_len(settings.seek).ok(); } + + Self::prepare_file(dst, settings) + } + + fn prepare_file(dst: File, settings: &'a Settings) -> UResult { let density = if settings.oconv.sparse { Density::Sparse } else { @@ -710,6 +774,24 @@ impl<'a> Output<'a> { Ok(Self { dst, settings }) } + /// Instantiate this struct with file descriptor as a destination. + /// + /// This is useful e.g. for the case when the file descriptor was + /// already opened by the system (stdout) and has a state + /// (current position) that shall be used. + fn new_file_from_stdout(settings: &'a Settings) -> UResult { + let fx = OwnedFileDescriptorOrHandle::from(io::stdout())?; + #[cfg(any(target_os = "linux", target_os = "android"))] + if let Some(libc_flags) = make_linux_oflags(&settings.oflags) { + nix::fcntl::fcntl( + fx.as_raw().as_raw_fd(), + F_SETFL(OFlag::from_bits_retain(libc_flags)), + )?; + } + + Self::prepare_file(fx.into_file(), settings) + } + /// Instantiate this struct with the given named pipe as a destination. #[cfg(unix)] fn new_fifo(filename: &Path, settings: &'a Settings) -> UResult { @@ -746,22 +828,45 @@ impl<'a> Output<'a> { /// the output file is no longer needed. If not possible, then /// this function prints an error message to stderr and sets the /// exit status code to 1. - #[allow(unused_variables)] + #[cfg_attr(not(target_os = "linux"), allow(clippy::unused_self, unused_variables))] fn discard_cache(&self, offset: libc::off_t, len: libc::off_t) { #[cfg(target_os = "linux")] { - show_if_err!(self - .dst - .discard_cache(offset, len) - .map_err_context(|| "failed to discard cache for: 'standard output'".to_string())); + show_if_err!(self.dst.discard_cache(offset, len).map_err_context(|| { + "failed to discard cache for: 'standard output'".to_string() + })); } - #[cfg(target_os = "linux")] + #[cfg(not(target_os = "linux"))] { // TODO Is there a way to discard filesystem cache on // these other operating systems? } } + /// writes a block of data. optionally retries when first try didn't complete + /// + /// this is needed by gnu-test: tests/dd/stats.s + /// the write can be interrupted by a system signal. + /// e.g. SIGUSR1 which is send to report status + /// without retry, the data might not be fully written to destination. + fn write_block(&mut self, chunk: &[u8]) -> io::Result { + let full_len = chunk.len(); + let mut base_idx = 0; + loop { + match self.dst.write(&chunk[base_idx..]) { + Ok(wlen) => { + base_idx += wlen; + // take iflags.fullblock as oflags shall not have this option + if (base_idx >= full_len) || !self.settings.iflags.fullblock { + return Ok(base_idx); + } + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + } + } + /// Write the given bytes one block at a time. /// /// This may write partial blocks (for example, if the underlying @@ -775,7 +880,7 @@ impl<'a> Output<'a> { let mut bytes_total = 0; for chunk in buf.chunks(self.settings.obs) { - let wlen = self.dst.write(chunk)?; + let wlen = self.write_block(chunk)?; if wlen < self.settings.obs { writes_partial += 1; } else { @@ -792,7 +897,7 @@ impl<'a> Output<'a> { } /// Flush the output to disk, if configured to do so. - fn sync(&mut self) -> std::io::Result<()> { + fn sync(&mut self) -> io::Result<()> { if self.settings.oconv.fsync { self.dst.fsync() } else if self.settings.oconv.fdatasync { @@ -804,7 +909,7 @@ impl<'a> Output<'a> { } /// Truncate the underlying file to the current stream position, if possible. - fn truncate(&mut self) -> std::io::Result<()> { + fn truncate(&mut self) -> io::Result<()> { self.dst.truncate() } } @@ -822,7 +927,7 @@ enum BlockWriter<'a> { Unbuffered(Output<'a>), } -impl<'a> BlockWriter<'a> { +impl BlockWriter<'_> { fn discard_cache(&self, offset: libc::off_t, len: libc::off_t) { match self { Self::Unbuffered(o) => o.discard_cache(offset, len), @@ -858,7 +963,7 @@ impl<'a> BlockWriter<'a> { }; } - fn write_blocks(&mut self, buf: &[u8]) -> std::io::Result { + fn write_blocks(&mut self, buf: &[u8]) -> io::Result { match self { Self::Unbuffered(o) => o.write_blocks(buf), Self::Buffered(o) => o.write_blocks(buf), @@ -866,6 +971,29 @@ impl<'a> BlockWriter<'a> { } } +/// depending on the command line arguments, this function +/// informs the OS to flush/discard the caches for input and/or output file. +fn flush_caches_full_length(i: &Input, o: &Output) -> io::Result<()> { + // TODO Better error handling for overflowing `len`. + if i.settings.iflags.nocache { + let offset = 0; + #[allow(clippy::useless_conversion)] + let len = i.src.len()?.try_into().unwrap(); + i.discard_cache(offset, len); + } + // Similarly, discard the system cache for the output file. + // + // TODO Better error handling for overflowing `len`. + if i.settings.oflags.nocache { + let offset = 0; + #[allow(clippy::useless_conversion)] + let len = o.dst.len()?.try_into().unwrap(); + o.discard_cache(offset, len); + } + + Ok(()) +} + /// Copy the given input data to this output, consuming both. /// /// This method contains the main loop for the `dd` program. Bytes @@ -877,7 +1005,7 @@ impl<'a> BlockWriter<'a> { /// /// If there is a problem reading from the input or writing to /// this output. -fn dd_copy(mut i: Input, o: Output) -> std::io::Result<()> { +fn dd_copy(mut i: Input, o: Output) -> io::Result<()> { // The read and write statistics. // // These objects are counters, initialized to zero. After each @@ -925,22 +1053,7 @@ fn dd_copy(mut i: Input, o: Output) -> std::io::Result<()> { // requests that we inform the system that we no longer // need the contents of the input file in a system cache. // - // TODO Better error handling for overflowing `len`. - if i.settings.iflags.nocache { - let offset = 0; - #[allow(clippy::useless_conversion)] - let len = i.src.len()?.try_into().unwrap(); - i.discard_cache(offset, len); - } - // Similarly, discard the system cache for the output file. - // - // TODO Better error handling for overflowing `len`. - if i.settings.oflags.nocache { - let offset = 0; - #[allow(clippy::useless_conversion)] - let len = o.dst.len()?.try_into().unwrap(); - o.discard_cache(offset, len); - } + flush_caches_full_length(&i, &o)?; return finalize( BlockWriter::Unbuffered(o), rstat, @@ -962,6 +1075,18 @@ fn dd_copy(mut i: Input, o: Output) -> std::io::Result<()> { // This avoids the need to query the OS monotonic clock for every block. let alarm = Alarm::with_interval(Duration::from_secs(1)); + // The signal handler spawns an own thread that waits for signals. + // When the signal is received, it calls a handler function. + // We inject a handler function that manually triggers the alarm. + #[cfg(target_os = "linux")] + let signal_handler = progress::SignalHandler::install_signal_handler(alarm.manual_trigger_fn()); + #[cfg(target_os = "linux")] + if let Err(e) = &signal_handler { + if Some(StatusLevel::None) != i.settings.status { + eprintln!("Internal dd Warning: Unable to register signal handler \n\t{e}"); + } + } + // Index in the input file where we are reading bytes and in // the output file where we are writing bytes. // @@ -985,13 +1110,13 @@ fn dd_copy(mut i: Input, o: Output) -> std::io::Result<()> { // blocks to this output. Read/write statistics are updated on // each iteration and cumulative statistics are reported to // the progress reporting thread. - while below_count_limit(&i.settings.count, &rstat) { + while below_count_limit(i.settings.count, &rstat) { // Read a block from the input then write the block to the output. // // As an optimization, make an educated guess about the // best buffer size for reading based on the number of // blocks already read and the number of blocks remaining. - let loop_bsize = calc_loop_bsize(&i.settings.count, &rstat, &wstat, i.settings.ibs, bsize); + let loop_bsize = calc_loop_bsize(i.settings.count, &rstat, &wstat, i.settings.ibs, bsize); let rstat_update = read_helper(&mut i, &mut buf, loop_bsize)?; if rstat_update.is_empty() { break; @@ -1030,11 +1155,20 @@ fn dd_copy(mut i: Input, o: Output) -> std::io::Result<()> { // error. rstat += rstat_update; wstat += wstat_update; - if alarm.is_triggered() { - let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), false); - prog_tx.send(prog_update).unwrap_or(()); + match alarm.get_trigger() { + ALARM_TRIGGER_NONE => {} + t @ (ALARM_TRIGGER_TIMER | ALARM_TRIGGER_SIGNAL) => { + let tp = match t { + ALARM_TRIGGER_TIMER => ProgUpdateType::Periodic, + _ => ProgUpdateType::Signal, + }; + let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), tp); + prog_tx.send(prog_update).unwrap_or(()); + } + _ => {} } } + finalize(o, rstat, wstat, start, &prog_tx, output_thread, truncate) } @@ -1047,7 +1181,7 @@ fn finalize( prog_tx: &mpsc::Sender, output_thread: thread::JoinHandle, truncate: bool, -) -> std::io::Result<()> { +) -> io::Result<()> { // Flush the output in case a partial write has been buffered but // not yet written. let wstat_update = output.flush()?; @@ -1062,12 +1196,13 @@ fn finalize( // Print the final read/write statistics. let wstat = wstat + wstat_update; - let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), true); + let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), ProgUpdateType::Final); prog_tx.send(prog_update).unwrap_or(()); // Wait for the output thread to finish output_thread .join() .expect("Failed to join with the output thread."); + Ok(()) } @@ -1105,11 +1240,7 @@ fn make_linux_oflags(oflags: &OFlags) -> Option { flag |= libc::O_SYNC; } - if flag == 0 { - None - } else { - Some(flag) - } + if flag == 0 { None } else { Some(flag) } } /// Read from an input (that is, a source of bytes) into the given buffer. @@ -1118,7 +1249,7 @@ fn make_linux_oflags(oflags: &OFlags) -> Option { /// `conv=swab` or `conv=block` command-line arguments. This function /// mutates the `buf` argument in-place. The returned [`ReadStat`] /// indicates how many blocks were read. -fn read_helper(i: &mut Input, buf: &mut Vec, bsize: usize) -> std::io::Result { +fn read_helper(i: &mut Input, buf: &mut Vec, bsize: usize) -> io::Result { // Local Helper Fns ------------------------------------------------- fn perform_swab(buf: &mut [u8]) { for base in (1..buf.len()).step_by(2) { @@ -1168,7 +1299,7 @@ fn calc_bsize(ibs: usize, obs: usize) -> usize { // Calculate the buffer size appropriate for this loop iteration, respecting // a count=N if present. fn calc_loop_bsize( - count: &Option, + count: Option, rstat: &ReadStat, wstat: &WriteStat, ibs: usize, @@ -1181,7 +1312,7 @@ fn calc_loop_bsize( cmp::min(ideal_bsize as u64, rremain * ibs as u64) as usize } Some(Num::Bytes(bmax)) => { - let bmax: u128 = (*bmax).into(); + let bmax: u128 = bmax.into(); let bremain: u128 = bmax - wstat.bytes_total; cmp::min(ideal_bsize as u128, bremain) as usize } @@ -1191,10 +1322,10 @@ fn calc_loop_bsize( // Decide if the current progress is below a count=N limit or return // true if no such limit is set. -fn below_count_limit(count: &Option, rstat: &ReadStat) -> bool { +fn below_count_limit(count: Option, rstat: &ReadStat) -> bool { match count { - Some(Num::Blocks(n)) => rstat.reads_complete + rstat.reads_partial < *n, - Some(Num::Bytes(n)) => rstat.bytes_total < *n, + Some(Num::Blocks(n)) => rstat.reads_complete + rstat.reads_partial < n, + Some(Num::Bytes(n)) => rstat.bytes_total < n, None => true, } } @@ -1270,11 +1401,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let matches = uu_app().try_get_matches_from(args)?; let settings: Settings = Parser::new().parse( - &matches + matches .get_many::(options::OPERANDS) - .unwrap_or_default() - .map(|s| s.as_ref()) - .collect::>()[..], + .unwrap_or_default(), )?; let i = match settings.infile { @@ -1287,9 +1416,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { #[cfg(unix)] Some(ref outfile) if is_fifo(outfile) => Output::new_fifo(Path::new(&outfile), &settings)?, Some(ref outfile) => Output::new_file(Path::new(&outfile), &settings)?, - None if is_stdout_redirected_to_seekable_file() => { - Output::new_file(Path::new(&stdout_canonicalized()), &settings)? - } + None if is_stdout_redirected_to_seekable_file() => Output::new_file_from_stdout(&settings)?, None => Output::new_stdout(&settings)?, }; dd_copy(i, o).map_err_context(|| "IO error".to_string()) @@ -1297,7 +1424,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .after_help(AFTER_HELP) @@ -1307,7 +1434,7 @@ pub fn uu_app() -> Command { #[cfg(test)] mod tests { - use crate::{calc_bsize, Output, Parser}; + use crate::{Output, Parser, calc_bsize}; use std::path::Path; @@ -1315,8 +1442,8 @@ mod tests { fn bsize_test_primes() { let (n, m) = (7901, 7919); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, n * m); } @@ -1325,8 +1452,8 @@ mod tests { fn bsize_test_rel_prime_obs_greater() { let (n, m) = (7 * 5119, 13 * 5119); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, 7 * 13 * 5119); } @@ -1335,8 +1462,8 @@ mod tests { fn bsize_test_rel_prime_ibs_greater() { let (n, m) = (13 * 5119, 7 * 5119); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, 7 * 13 * 5119); } @@ -1345,8 +1472,8 @@ mod tests { fn bsize_test_3fac_rel_prime() { let (n, m) = (11 * 13 * 5119, 7 * 11 * 5119); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, 7 * 11 * 13 * 5119); } @@ -1355,8 +1482,8 @@ mod tests { fn bsize_test_ibs_greater() { let (n, m) = (512 * 1024, 256 * 1024); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, n); } @@ -1365,8 +1492,8 @@ mod tests { fn bsize_test_obs_greater() { let (n, m) = (256 * 1024, 512 * 1024); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, m); } @@ -1375,8 +1502,8 @@ mod tests { fn bsize_test_bs_eq() { let (n, m) = (1024, 1024); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, m); } diff --git a/src/uu/dd/src/numbers.rs b/src/uu/dd/src/numbers.rs index 8a6fa5a7a37..b66893d8d35 100644 --- a/src/uu/dd/src/numbers.rs +++ b/src/uu/dd/src/numbers.rs @@ -2,7 +2,8 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -/// Functions for formatting a number as a magnitude and a unit suffix. + +//! Functions for formatting a number as a magnitude and a unit suffix. /// The first ten powers of 1024. const IEC_BASES: [u128; 10] = [ @@ -82,14 +83,14 @@ pub(crate) fn to_magnitude_and_suffix(n: u128, suffix_type: SuffixType) -> Strin if quotient < 10.0 { format!("{quotient:.1} {suffix}") } else { - format!("{} {}", quotient.round(), suffix) + format!("{} {suffix}", quotient.round()) } } #[cfg(test)] mod tests { - use crate::numbers::{to_magnitude_and_suffix, SuffixType}; + use crate::numbers::{SuffixType, to_magnitude_and_suffix}; #[test] fn test_to_magnitude_and_suffix_powers_of_1024() { diff --git a/src/uu/dd/src/parseargs.rs b/src/uu/dd/src/parseargs.rs index 60ce9a6971f..dd9a53fd884 100644 --- a/src/uu/dd/src/parseargs.rs +++ b/src/uu/dd/src/parseargs.rs @@ -9,28 +9,42 @@ mod unit_tests; use super::{ConversionMode, IConvFlags, IFlags, Num, OConvFlags, OFlags, Settings, StatusLevel}; use crate::conversion_tables::ConversionTable; -use std::error::Error; +use thiserror::Error; use uucore::display::Quotable; use uucore::error::UError; -use uucore::parse_size::{ParseSizeError, Parser as SizeParser}; +use uucore::parser::parse_size::{ParseSizeError, Parser as SizeParser}; use uucore::show_warning; /// Parser Errors describe errors with parser input -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Error)] pub enum ParseError { + #[error("Unrecognized operand '{0}'")] UnrecognizedOperand(String), + #[error("Only one of conv=ascii conv=ebcdic or conv=ibm may be specified")] MultipleFmtTable, + #[error("Only one of conv=lcase or conv=ucase may be specified")] MultipleUCaseLCase, + #[error("Only one of conv=block or conv=unblock may be specified")] MultipleBlockUnblock, + #[error("Only one ov conv=excl or conv=nocreat may be specified")] MultipleExclNoCreate, + #[error("invalid input flag: ‘{}’\nTry '{} --help' for more information.", .0, uucore::execution_phrase())] FlagNoMatch(String), + #[error("Unrecognized conv=CONV -> {0}")] ConvFlagNoMatch(String), + #[error("invalid number: ‘{0}’")] MultiplierStringParseFailure(String), + #[error("Multiplier string would overflow on current system -> {0}")] MultiplierStringOverflow(String), + #[error("conv=block or conv=unblock specified without cbs=N")] BlockUnblockWithoutCBS, + #[error("status=LEVEL not recognized -> {0}")] StatusLevelNotRecognized(String), + #[error("feature not implemented on this system -> {0}")] Unimplemented(String), + #[error("{0}=N cannot fit into memory")] BsOutOfRange(String), + #[error("invalid number: ‘{0}’")] InvalidNumber(String), } @@ -112,13 +126,19 @@ impl Parser { Self::default() } - pub(crate) fn parse(self, operands: &[&str]) -> Result { + pub(crate) fn parse( + self, + operands: impl IntoIterator>, + ) -> Result { self.read(operands)?.validate() } - pub(crate) fn read(mut self, operands: &[&str]) -> Result { + pub(crate) fn read( + mut self, + operands: impl IntoIterator>, + ) -> Result { for operand in operands { - self.parse_operand(operand)?; + self.parse_operand(operand.as_ref())?; } Ok(self) @@ -275,7 +295,7 @@ impl Parser { fn parse_n(val: &str) -> Result { let n = parse_bytes_with_opt_multiplier(val)?; - Ok(if val.ends_with('B') { + Ok(if val.contains('B') { Num::Bytes(n) } else { Num::Blocks(n) @@ -396,69 +416,6 @@ impl Parser { } } -impl std::fmt::Display for ParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::UnrecognizedOperand(arg) => { - write!(f, "Unrecognized operand '{arg}'") - } - Self::MultipleFmtTable => { - write!( - f, - "Only one of conv=ascii conv=ebcdic or conv=ibm may be specified" - ) - } - Self::MultipleUCaseLCase => { - write!(f, "Only one of conv=lcase or conv=ucase may be specified") - } - Self::MultipleBlockUnblock => { - write!(f, "Only one of conv=block or conv=unblock may be specified") - } - Self::MultipleExclNoCreate => { - write!(f, "Only one ov conv=excl or conv=nocreat may be specified") - } - Self::FlagNoMatch(arg) => { - // Additional message about 'dd --help' is displayed only in this situation. - write!( - f, - "invalid input flag: ‘{}’\nTry '{} --help' for more information.", - arg, - uucore::execution_phrase() - ) - } - Self::ConvFlagNoMatch(arg) => { - write!(f, "Unrecognized conv=CONV -> {arg}") - } - Self::MultiplierStringParseFailure(arg) => { - write!(f, "Unrecognized byte multiplier -> {arg}") - } - Self::MultiplierStringOverflow(arg) => { - write!( - f, - "Multiplier string would overflow on current system -> {arg}" - ) - } - Self::BlockUnblockWithoutCBS => { - write!(f, "conv=block or conv=unblock specified without cbs=N") - } - Self::StatusLevelNotRecognized(arg) => { - write!(f, "status=LEVEL not recognized -> {arg}") - } - Self::BsOutOfRange(arg) => { - write!(f, "{arg}=N cannot fit into memory") - } - Self::Unimplemented(arg) => { - write!(f, "feature not implemented on this system -> {arg}") - } - Self::InvalidNumber(arg) => { - write!(f, "invalid number: ‘{arg}’") - } - } - } -} - -impl Error for ParseError {} - impl UError for ParseError { fn code(&self) -> i32 { 1 @@ -474,8 +431,9 @@ fn show_zero_multiplier_warning() { } /// Parse bytes using str::parse, then map error if needed. -fn parse_bytes_only(s: &str) -> Result { - s.parse() +fn parse_bytes_only(s: &str, i: usize) -> Result { + s[..i] + .parse() .map_err(|_| ParseError::MultiplierStringParseFailure(s.to_string())) } @@ -490,6 +448,8 @@ fn parse_bytes_only(s: &str) -> Result { /// 512. You can also use standard block size suffixes like `'k'` for /// 1024. /// +/// If the number would be too large, return [`u64::MAX`] instead. +/// /// # Errors /// /// If a number cannot be parsed or if the multiplication would cause @@ -507,21 +467,18 @@ fn parse_bytes_only(s: &str) -> Result { fn parse_bytes_no_x(full: &str, s: &str) -> Result { let parser = SizeParser { capital_b_bytes: true, + no_empty_numeric: true, ..Default::default() }; let (num, multiplier) = match (s.find('c'), s.rfind('w'), s.rfind('b')) { (None, None, None) => match parser.parse_u64(s) { Ok(n) => (n, 1), - Err(ParseSizeError::InvalidSuffix(_) | ParseSizeError::ParseFailure(_)) => { - return Err(ParseError::InvalidNumber(full.to_string())) - } - Err(ParseSizeError::SizeTooBig(_)) => { - return Err(ParseError::MultiplierStringOverflow(full.to_string())) - } + Err(ParseSizeError::SizeTooBig(_)) => (u64::MAX, 1), + Err(_) => return Err(ParseError::InvalidNumber(full.to_string())), }, - (Some(i), None, None) => (parse_bytes_only(&s[..i])?, 1), - (None, Some(i), None) => (parse_bytes_only(&s[..i])?, 2), - (None, None, Some(i)) => (parse_bytes_only(&s[..i])?, 512), + (Some(i), None, None) => (parse_bytes_only(s, i)?, 1), + (None, Some(i), None) => (parse_bytes_only(s, i)?, 2), + (None, None, Some(i)) => (parse_bytes_only(s, i)?, 512), _ => return Err(ParseError::MultiplierStringParseFailure(full.to_string())), }; num.checked_mul(multiplier) @@ -630,22 +587,42 @@ fn conversion_mode( #[cfg(test)] mod tests { - use crate::parseargs::parse_bytes_with_opt_multiplier; + use crate::Num; + use crate::parseargs::{Parser, parse_bytes_with_opt_multiplier}; + use std::matches; + const BIG: &str = "9999999999999999999999999999999999999999999999999999999999999"; #[test] - fn test_parse_bytes_with_opt_multiplier() { + fn test_parse_bytes_with_opt_multiplier_invalid() { + assert!(parse_bytes_with_opt_multiplier("123asdf").is_err()); + } + + #[test] + fn test_parse_bytes_with_opt_multiplier_without_x() { assert_eq!(parse_bytes_with_opt_multiplier("123").unwrap(), 123); assert_eq!(parse_bytes_with_opt_multiplier("123c").unwrap(), 123); // 123 * 1 assert_eq!(parse_bytes_with_opt_multiplier("123w").unwrap(), 123 * 2); assert_eq!(parse_bytes_with_opt_multiplier("123b").unwrap(), 123 * 512); - assert_eq!(parse_bytes_with_opt_multiplier("123x3").unwrap(), 123 * 3); assert_eq!(parse_bytes_with_opt_multiplier("123k").unwrap(), 123 * 1024); - assert_eq!(parse_bytes_with_opt_multiplier("1x2x3").unwrap(), 6); // 1 * 2 * 3 + assert_eq!(parse_bytes_with_opt_multiplier(BIG).unwrap(), u64::MAX); + } + #[test] + fn test_parse_bytes_with_opt_multiplier_with_x() { + assert_eq!(parse_bytes_with_opt_multiplier("123x3").unwrap(), 123 * 3); + assert_eq!(parse_bytes_with_opt_multiplier("1x2x3").unwrap(), 6); // 1 * 2 * 3 assert_eq!( parse_bytes_with_opt_multiplier("1wx2cx3w").unwrap(), 2 * 2 * (3 * 2) // (1 * 2) * (2 * 1) * (3 * 2) ); - assert!(parse_bytes_with_opt_multiplier("123asdf").is_err()); + } + #[test] + fn test_parse_n() { + for arg in ["1x8x4", "1c", "123b", "123w"] { + assert!(matches!(Parser::parse_n(arg), Ok(Num::Blocks(_)))); + } + for arg in ["1Bx8x4", "2Bx8", "2Bx8B", "2x8B"] { + assert!(matches!(Parser::parse_n(arg), Ok(Num::Bytes(_)))); + } } } diff --git a/src/uu/dd/src/parseargs/unit_tests.rs b/src/uu/dd/src/parseargs/unit_tests.rs index 51b0933e926..cde0ef0cc1f 100644 --- a/src/uu/dd/src/parseargs/unit_tests.rs +++ b/src/uu/dd/src/parseargs/unit_tests.rs @@ -6,13 +6,14 @@ use super::*; +use crate::StatusLevel; use crate::conversion_tables::{ ASCII_TO_EBCDIC_UCASE_TO_LCASE, ASCII_TO_IBM, EBCDIC_TO_ASCII_LCASE_TO_UCASE, }; use crate::parseargs::Parser; -use crate::StatusLevel; #[cfg(not(any(target_os = "linux", target_os = "android")))] +#[allow(clippy::useless_vec)] #[test] fn unimplemented_flags_should_error_non_linux() { let mut succeeded = Vec::new(); @@ -28,54 +29,38 @@ fn unimplemented_flags_should_error_non_linux() { "noctty", "nofollow", ] { - let args = vec![format!("iflag={}", flag)]; - - if Parser::new() - .parse(&args.iter().map(AsRef::as_ref).collect::>()[..]) - .is_ok() - { - succeeded.push(format!("iflag={}", flag)); + let arg = format!("iflag={flag}"); + if Parser::new().parse([&arg]).is_ok() { + succeeded.push(arg); } - let args = vec![format!("oflag={}", flag)]; - - if Parser::new() - .parse(&args.iter().map(AsRef::as_ref).collect::>()[..]) - .is_ok() - { - succeeded.push(format!("iflag={}", flag)); + let arg = format!("oflag={flag}"); + if Parser::new().parse([&arg]).is_ok() { + succeeded.push(arg); } } assert!( succeeded.is_empty(), - "The following flags did not panic as expected: {:?}", - succeeded + "The following flags did not panic as expected: {succeeded:?}", ); } #[test] +#[allow(clippy::useless_vec)] fn unimplemented_flags_should_error() { let mut succeeded = Vec::new(); // The following flags are not implemented for flag in ["cio", "nolinks", "text", "binary"] { - let args = vec![format!("iflag={flag}")]; - - if Parser::new() - .parse(&args.iter().map(AsRef::as_ref).collect::>()[..]) - .is_ok() - { - succeeded.push(format!("iflag={flag}")); + let arg = format!("iflag={flag}"); + if Parser::new().parse([&arg]).is_ok() { + succeeded.push(arg); } - let args = vec![format!("oflag={flag}")]; - - if Parser::new() - .parse(&args.iter().map(AsRef::as_ref).collect::>()[..]) - .is_ok() - { - succeeded.push(format!("iflag={flag}")); + let arg = format!("oflag={flag}"); + if Parser::new().parse([&arg]).is_ok() { + succeeded.push(arg); } } @@ -87,14 +72,14 @@ fn unimplemented_flags_should_error() { #[test] fn test_status_level_absent() { - let args = &["if=foo.file", "of=bar.file"]; + let args = ["if=foo.file", "of=bar.file"]; assert_eq!(Parser::new().parse(args).unwrap().status, None); } #[test] fn test_status_level_none() { - let args = &["status=none", "if=foo.file", "of=bar.file"]; + let args = ["status=none", "if=foo.file", "of=bar.file"]; assert_eq!( Parser::new().parse(args).unwrap().status, @@ -105,7 +90,7 @@ fn test_status_level_none() { #[test] #[allow(clippy::cognitive_complexity)] fn test_all_top_level_args_no_leading_dashes() { - let args = &[ + let args = [ "if=foo.file", "of=bar.file", "ibs=10", @@ -155,7 +140,7 @@ fn test_all_top_level_args_no_leading_dashes() { ); // no conv flags apply to output - assert_eq!(settings.oconv, OConvFlags::default(),); + assert_eq!(settings.oconv, OConvFlags::default()); // iconv=count_bytes,skip_bytes assert_eq!( @@ -180,7 +165,7 @@ fn test_all_top_level_args_no_leading_dashes() { #[test] fn test_status_level_progress() { - let args = &["if=foo.file", "of=bar.file", "status=progress"]; + let args = ["if=foo.file", "of=bar.file", "status=progress"]; let settings = Parser::new().parse(args).unwrap(); @@ -189,7 +174,7 @@ fn test_status_level_progress() { #[test] fn test_status_level_noxfer() { - let args = &["if=foo.file", "status=noxfer", "of=bar.file"]; + let args = ["if=foo.file", "status=noxfer", "of=bar.file"]; let settings = Parser::new().parse(args).unwrap(); @@ -198,7 +183,7 @@ fn test_status_level_noxfer() { #[test] fn test_multiple_flags_options() { - let args = &[ + let args = [ "iflag=fullblock,count_bytes", "iflag=skip_bytes", "oflag=append", @@ -245,7 +230,7 @@ fn test_multiple_flags_options() { #[test] fn test_override_multiple_options() { - let args = &[ + let args = [ "if=foo.file", "if=correct.file", "of=bar.file", @@ -287,31 +272,31 @@ fn test_override_multiple_options() { #[test] fn icf_ctable_error() { - let args = &["conv=ascii,ebcdic,ibm"]; + let args = ["conv=ascii,ebcdic,ibm"]; assert!(Parser::new().parse(args).is_err()); } #[test] fn icf_case_error() { - let args = &["conv=ucase,lcase"]; + let args = ["conv=ucase,lcase"]; assert!(Parser::new().parse(args).is_err()); } #[test] fn icf_block_error() { - let args = &["conv=block,unblock"]; + let args = ["conv=block,unblock"]; assert!(Parser::new().parse(args).is_err()); } #[test] fn icf_creat_error() { - let args = &["conv=excl,nocreat"]; + let args = ["conv=excl,nocreat"]; assert!(Parser::new().parse(args).is_err()); } #[test] fn parse_icf_token_ibm() { - let args = &["conv=ibm"]; + let args = ["conv=ibm"]; let settings = Parser::new().parse(args).unwrap(); assert_eq!( @@ -325,7 +310,7 @@ fn parse_icf_token_ibm() { #[test] fn parse_icf_tokens_elu() { - let args = &["conv=ebcdic,lcase"]; + let args = ["conv=ebcdic,lcase"]; let settings = Parser::new().parse(args).unwrap(); assert_eq!( @@ -339,7 +324,9 @@ fn parse_icf_tokens_elu() { #[test] fn parse_icf_tokens_remaining() { - let args = &["conv=ascii,ucase,block,sparse,swab,sync,noerror,excl,nocreat,notrunc,noerror,fdatasync,fsync"]; + let args = [ + "conv=ascii,ucase,block,sparse,swab,sync,noerror,excl,nocreat,notrunc,noerror,fdatasync,fsync", + ]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -366,7 +353,7 @@ fn parse_icf_tokens_remaining() { #[test] fn parse_iflag_tokens() { - let args = &["iflag=fullblock,count_bytes,skip_bytes"]; + let args = ["iflag=fullblock,count_bytes,skip_bytes"]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -383,7 +370,7 @@ fn parse_iflag_tokens() { #[test] fn parse_oflag_tokens() { - let args = &["oflag=append,seek_bytes"]; + let args = ["oflag=append,seek_bytes"]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -400,7 +387,7 @@ fn parse_oflag_tokens() { #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn parse_iflag_tokens_linux() { - let args = &["iflag=direct,directory,dsync,sync,nonblock,noatime,noctty,nofollow"]; + let args = ["iflag=direct,directory,dsync,sync,nonblock,noatime,noctty,nofollow"]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -423,7 +410,7 @@ fn parse_iflag_tokens_linux() { #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn parse_oflag_tokens_linux() { - let args = &["oflag=direct,directory,dsync,sync,nonblock,noatime,noctty,nofollow"]; + let args = ["oflag=direct,directory,dsync,sync,nonblock,noatime,noctty,nofollow"]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -506,14 +493,6 @@ mod test_64bit_arch { ); } -#[test] -#[should_panic] -fn test_overflow_panic() { - let bs_str = format!("{}KiB", u64::MAX); - - parse_bytes_with_opt_multiplier(&bs_str).unwrap(); -} - #[test] #[should_panic] fn test_neg_panic() { diff --git a/src/uu/dd/src/progress.rs b/src/uu/dd/src/progress.rs index 238b2ab39a2..85f5fa85af8 100644 --- a/src/uu/dd/src/progress.rs +++ b/src/uu/dd/src/progress.rs @@ -11,27 +11,25 @@ //! updater that runs in its own thread. use std::io::Write; use std::sync::mpsc; +#[cfg(target_os = "linux")] +use std::thread::JoinHandle; use std::time::Duration; +#[cfg(target_os = "linux")] +use signal_hook::iterator::Handle; use uucore::{ error::UResult, format::num_format::{FloatVariant, Formatter}, }; -use crate::numbers::{to_magnitude_and_suffix, SuffixType}; +use crate::numbers::{SuffixType, to_magnitude_and_suffix}; -// On Linux, we register a signal handler that prints progress updates. -#[cfg(target_os = "linux")] -use signal_hook::consts::signal; -#[cfg(target_os = "linux")] -use std::{ - env, - error::Error, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, -}; +#[derive(PartialEq, Eq)] +pub(crate) enum ProgUpdateType { + Periodic, + Signal, + Final, +} /// Summary statistics for read and write progress of dd for a given duration. pub(crate) struct ProgUpdate { @@ -53,7 +51,7 @@ pub(crate) struct ProgUpdate { /// The status of the write. /// /// True if the write is completed, false if still in-progress. - pub(crate) complete: bool, + pub(crate) update_type: ProgUpdateType, } impl ProgUpdate { @@ -62,13 +60,13 @@ impl ProgUpdate { read_stat: ReadStat, write_stat: WriteStat, duration: Duration, - complete: bool, + update_type: ProgUpdateType, ) -> Self { Self { read_stat, write_stat, duration, - complete, + update_type, } } @@ -159,7 +157,7 @@ impl ProgUpdate { variant: FloatVariant::Shortest, ..Default::default() } - .fmt(&mut duration_str, duration)?; + .fmt(&mut duration_str, &duration.into())?; // We assume that printf will output valid UTF-8 let duration_str = std::str::from_utf8(&duration_str).unwrap(); @@ -433,7 +431,7 @@ pub(crate) fn gen_prog_updater( let mut progress_printed = false; while let Ok(update) = rx.recv() { // Print the final read/write statistics. - if update.complete { + if update.update_type == ProgUpdateType::Final { update.print_final_stats(print_level, progress_printed); return; } @@ -445,6 +443,49 @@ pub(crate) fn gen_prog_updater( } } +/// signal handler listens for SIGUSR1 signal and runs provided closure. +#[cfg(target_os = "linux")] +pub(crate) struct SignalHandler { + handle: Handle, + thread: Option>, +} + +#[cfg(target_os = "linux")] +impl SignalHandler { + pub(crate) fn install_signal_handler( + f: Box, + ) -> Result { + use signal_hook::consts::signal::*; + use signal_hook::iterator::Signals; + + let mut signals = Signals::new([SIGUSR1])?; + let handle = signals.handle(); + let thread = std::thread::spawn(move || { + for signal in &mut signals { + match signal { + SIGUSR1 => (*f)(), + _ => unreachable!(), + } + } + }); + + Ok(Self { + handle, + thread: Some(thread), + }) + } +} + +#[cfg(target_os = "linux")] +impl Drop for SignalHandler { + fn drop(&mut self) { + self.handle.close(); + if let Some(thread) = std::mem::take(&mut self.thread) { + thread.join().unwrap(); + } + } +} + /// Return a closure that can be used in its own thread to print progress info. /// /// This function returns a closure that receives [`ProgUpdate`] @@ -459,50 +500,31 @@ pub(crate) fn gen_prog_updater( rx: mpsc::Receiver, print_level: Option, ) -> impl Fn() { - // TODO: SIGINFO: Trigger progress line reprint. BSD-style Linux only. - const SIGUSR1_USIZE: usize = signal::SIGUSR1 as usize; - fn posixly_correct() -> bool { - env::var("POSIXLY_CORRECT").is_ok() - } - fn register_linux_signal_handler(sigval: Arc) -> Result<(), Box> { - if !posixly_correct() { - signal_hook::flag::register_usize(signal::SIGUSR1, sigval, SIGUSR1_USIZE)?; - } - - Ok(()) - } // -------------------------------------------------------------- move || { - let sigval = Arc::new(AtomicUsize::new(0)); - - register_linux_signal_handler(sigval.clone()).unwrap_or_else(|e| { - if Some(StatusLevel::None) != print_level { - eprintln!("Internal dd Warning: Unable to register signal handler \n\t{e}"); - } - }); - // Holds the state of whether we have printed the current progress. // This is needed so that we know whether or not to print a newline // character before outputting non-progress data. let mut progress_printed = false; while let Ok(update) = rx.recv() { - // Print the final read/write statistics. - if update.complete { - update.print_final_stats(print_level, progress_printed); - return; - } - // (Re)print status line if progress is requested. - if Some(StatusLevel::Progress) == print_level && !update.complete { - update.reprint_prog_line(); - progress_printed = true; - } - // Handle signals and set the signal to un-seen. - // This will print a maximum of 1 time per second, even though it - // should be printing on every SIGUSR1. - if let SIGUSR1_USIZE = sigval.swap(0, Ordering::Relaxed) { - update.print_transfer_stats(progress_printed); - // Reset the progress printed, since print_transfer_stats always prints a newline. - progress_printed = false; + match update.update_type { + ProgUpdateType::Final => { + // Print the final read/write statistics. + update.print_final_stats(print_level, progress_printed); + return; + } + ProgUpdateType::Periodic => { + // (Re)print status line if progress is requested. + if Some(StatusLevel::Progress) == print_level { + update.reprint_prog_line(); + progress_printed = true; + } + } + ProgUpdateType::Signal => { + update.print_transfer_stats(progress_printed); + // Reset the progress printed, since print_transfer_stats always prints a newline. + progress_printed = false; + } } } } @@ -524,7 +546,7 @@ mod tests { ..Default::default() }, duration: Duration::new(1, 0), // one second - complete: false, + update_type: super::ProgUpdateType::Periodic, } } @@ -533,7 +555,7 @@ mod tests { read_stat: ReadStat::default(), write_stat: WriteStat::default(), duration, - complete: false, + update_type: super::ProgUpdateType::Periodic, } } @@ -558,12 +580,12 @@ mod tests { let read_stat = ReadStat::new(1, 2, 3, 4); let write_stat = WriteStat::new(4, 5, 6); let duration = Duration::new(789, 0); - let complete = false; + let update_type = super::ProgUpdateType::Periodic; let prog_update = ProgUpdate { read_stat, write_stat, duration, - complete, + update_type, }; let mut cursor = Cursor::new(vec![]); @@ -580,7 +602,7 @@ mod tests { read_stat: ReadStat::default(), write_stat: WriteStat::default(), duration: Duration::new(1, 0), // one second - complete: false, + update_type: super::ProgUpdateType::Periodic, }; let mut cursor = Cursor::new(vec![]); @@ -636,7 +658,7 @@ mod tests { read_stat: ReadStat::default(), write_stat: WriteStat::default(), duration: Duration::new(1, 0), // one second - complete: false, + update_type: super::ProgUpdateType::Periodic, }; let mut cursor = Cursor::new(vec![]); prog_update @@ -657,7 +679,7 @@ mod tests { read_stat: ReadStat::default(), write_stat: WriteStat::default(), duration: Duration::new(1, 0), // one second - complete: false, + update_type: super::ProgUpdateType::Periodic, }; let mut cursor = Cursor::new(vec![]); let rewrite = true; diff --git a/src/uu/df/Cargo.toml b/src/uu/df/Cargo.toml index e9aa192e820..6585b0abc6a 100644 --- a/src/uu/df/Cargo.toml +++ b/src/uu/df/Cargo.toml @@ -1,26 +1,30 @@ [package] name = "uu_df" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "df ~ (uutils) display file system information" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/df" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/df.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true, features = ["libc", "fsext"] } +uucore = { workspace = true, features = ["libc", "fsext", "parser"] } unicode-width = { workspace = true } +thiserror = { workspace = true } [dev-dependencies] -tempfile = "3" +tempfile = { workspace = true } [[bin]] name = "df" diff --git a/src/uu/df/src/blocks.rs b/src/uu/df/src/blocks.rs index d7a689d8c86..26b763cac1a 100644 --- a/src/uu/df/src/blocks.rs +++ b/src/uu/df/src/blocks.rs @@ -9,7 +9,7 @@ use std::{env, fmt}; use uucore::{ display::Quotable, - parse_size::{parse_size_u64, ParseSizeError}, + parser::parse_size::{ParseSizeError, parse_size_u64}, }; /// The first ten powers of 1024. @@ -98,9 +98,9 @@ pub(crate) fn to_magnitude_and_suffix(n: u128, suffix_type: SuffixType) -> Strin if rem % (bases[i] / 10) == 0 { format!("{quot}.{tenths_place}{suffix}") } else if tenths_place + 1 == 10 || quot >= 10 { - format!("{}{}", quot + 1, suffix) + format!("{}{suffix}", quot + 1) } else { - format!("{}.{}{}", quot, tenths_place + 1, suffix) + format!("{quot}.{}{suffix}", tenths_place + 1) } } } @@ -184,11 +184,7 @@ pub(crate) fn read_block_size(matches: &ArgMatches) -> Result Option { for env_var in ["DF_BLOCK_SIZE", "BLOCK_SIZE", "BLOCKSIZE"] { if let Ok(env_size) = env::var(env_var) { - if let Ok(size) = parse_size_u64(&env_size) { - return Some(size); - } else { - return None; - } + return parse_size_u64(&env_size).ok(); } } @@ -216,7 +212,7 @@ mod tests { use std::env; - use crate::blocks::{to_magnitude_and_suffix, BlockSize, SuffixType}; + use crate::blocks::{BlockSize, SuffixType, to_magnitude_and_suffix}; #[test] fn test_to_magnitude_and_suffix_powers_of_1024() { @@ -294,8 +290,8 @@ mod tests { #[test] fn test_default_block_size() { assert_eq!(BlockSize::Bytes(1024), BlockSize::default()); - env::set_var("POSIXLY_CORRECT", "1"); + unsafe { env::set_var("POSIXLY_CORRECT", "1") }; assert_eq!(BlockSize::Bytes(512), BlockSize::default()); - env::remove_var("POSIXLY_CORRECT"); + unsafe { env::remove_var("POSIXLY_CORRECT") }; } } diff --git a/src/uu/df/src/columns.rs b/src/uu/df/src/columns.rs index 0659d7f7da7..0d2d121a3d1 100644 --- a/src/uu/df/src/columns.rs +++ b/src/uu/df/src/columns.rs @@ -4,7 +4,9 @@ // file that was distributed with this source code. // spell-checker:ignore itotal iused iavail ipcent pcent squashfs use crate::{OPT_INODES, OPT_OUTPUT, OPT_PRINT_TYPE}; -use clap::{parser::ValueSource, ArgMatches}; +use clap::{ArgMatches, parser::ValueSource}; +use thiserror::Error; +use uucore::display::Quotable; /// The columns in the output table produced by `df`. /// @@ -56,9 +58,10 @@ pub(crate) enum Column { } /// An error while defining which columns to display in the output table. -#[derive(Debug)] +#[derive(Debug, Error)] pub(crate) enum ColumnError { /// If a column appears more than once in the `--output` argument. + #[error("{}", .0.quote())] MultipleColumns(String), } @@ -201,6 +204,7 @@ impl Column { // 14 = length of "Filesystem" plus 4 spaces Self::Source => 14, Self::Used => 5, + Self::Size => 5, // the shortest headers have a length of 4 chars so we use that as the minimum width _ => 4, } diff --git a/src/uu/df/src/df.rs b/src/uu/df/src/df.rs index c21ba98471a..9e2bb6920af 100644 --- a/src/uu/df/src/df.rs +++ b/src/uu/df/src/df.rs @@ -12,22 +12,21 @@ use blocks::HumanReadable; use clap::builder::ValueParser; use table::HeaderMode; use uucore::display::Quotable; -use uucore::error::FromIo; -use uucore::error::{UError, UResult, USimpleError}; -use uucore::fsext::{read_fs_list, MountInfo}; -use uucore::parse_size::ParseSizeError; +use uucore::error::{UError, UResult, USimpleError, get_exit_code}; +use uucore::fsext::{MountInfo, read_fs_list}; +use uucore::parser::parse_size::ParseSizeError; use uucore::{format_usage, help_about, help_section, help_usage, show}; -use clap::{crate_version, parser::ValueSource, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command, parser::ValueSource}; -use std::error::Error; use std::ffi::OsString; -use std::fmt; use std::path::Path; +use thiserror::Error; -use crate::blocks::{read_block_size, BlockSize}; +use crate::blocks::{BlockSize, read_block_size}; use crate::columns::{Column, ColumnError}; use crate::filesystem::Filesystem; +use crate::filesystem::FsError; use crate::table::Table; const ABOUT: &str = help_about!("df.md"); @@ -114,52 +113,32 @@ impl Default for Options { } } -#[derive(Debug)] +#[derive(Debug, Error)] enum OptionsError { + // TODO This needs to vary based on whether `--block-size` + // or `-B` were provided. + #[error("--block-size argument '{0}' too large")] BlockSizeTooLarge(String), + // TODO This needs to vary based on whether `--block-size` + // or `-B` were provided., + #[error("invalid --block-size argument {0}")] InvalidBlockSize(String), + // TODO This needs to vary based on whether `--block-size` + // or `-B` were provided. + #[error("invalid suffix in --block-size argument {0}")] InvalidSuffix(String), /// An error getting the columns to display in the output table. + #[error("option --output: field {0} used more than once")] ColumnError(ColumnError), + #[error("{}", .0.iter() + .map(|t| format!("file system type {} both selected and excluded", t.quote())) + .collect::>() + .join(format!("\n{}: ", uucore::util_name()).as_str()))] FilesystemTypeBothSelectedAndExcluded(Vec), } -impl fmt::Display for OptionsError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - // TODO This needs to vary based on whether `--block-size` - // or `-B` were provided. - Self::BlockSizeTooLarge(s) => { - write!(f, "--block-size argument {} too large", s.quote()) - } - // TODO This needs to vary based on whether `--block-size` - // or `-B` were provided. - Self::InvalidBlockSize(s) => write!(f, "invalid --block-size argument {s}"), - // TODO This needs to vary based on whether `--block-size` - // or `-B` were provided. - Self::InvalidSuffix(s) => write!(f, "invalid suffix in --block-size argument {s}"), - Self::ColumnError(ColumnError::MultipleColumns(s)) => write!( - f, - "option --output: field {} used more than once", - s.quote() - ), - #[allow(clippy::print_in_format_impl)] - Self::FilesystemTypeBothSelectedAndExcluded(types) => { - for t in types { - eprintln!( - "{}: file system type {} both selected and excluded", - uucore::util_name(), - t.quote() - ); - } - Ok(()) - } - } - } -} - impl Options { /// Convert command-line arguments into [`Options`]. fn from(matches: &ArgMatches) -> Result { @@ -189,6 +168,7 @@ impl Options { .to_string(), ), ParseSizeError::ParseFailure(s) => OptionsError::InvalidBlockSize(s), + ParseSizeError::PhysicalMem(s) => OptionsError::InvalidBlockSize(s), })?, header_mode: { if matches.get_flag(OPT_HUMAN_READABLE_BINARY) @@ -242,7 +222,10 @@ fn is_included(mi: &MountInfo, opt: &Options) -> bool { } // Don't show pseudo filesystems unless `--all` has been given. - if mi.dummy && !opt.show_all_fs { + // The "lofs" filesystem is a loopback + // filesystem present on Solaris and FreeBSD systems. It + // is similar to a symbolic link. + if (mi.dummy || mi.fs_type == "lofs") && !opt.show_all_fs { return false; } @@ -305,35 +288,11 @@ fn is_best(previous: &[MountInfo], mi: &MountInfo) -> bool { true } -/// Keep only the specified subset of [`MountInfo`] instances. -/// -/// The `opt` argument specifies a variety of ways of excluding -/// [`MountInfo`] instances; see [`Options`] for more information. -/// -/// Finally, if there are duplicate entries, the one with the shorter -/// path is kept. - -fn filter_mount_list(vmi: Vec, opt: &Options) -> Vec { - let mut result = vec![]; - for mi in vmi { - // TODO The running time of the `is_best()` function is linear - // in the length of `result`. That makes the running time of - // this loop quadratic in the length of `vmi`. This could be - // improved by a more efficient implementation of `is_best()`, - // but `vmi` is probably not very long in practice. - if is_included(&mi, opt) && is_best(&result, &mi) { - result.push(mi); - } - } - result -} - /// Get all currently mounted filesystems. /// /// `opt` excludes certain filesystems from consideration and allows for the synchronization of filesystems before running; see /// [`Options`] for more information. - -fn get_all_filesystems(opt: &Options) -> Result, std::io::Error> { +fn get_all_filesystems(opt: &Options) -> UResult> { // Run a sync call before any operation if so instructed. if opt.sync { #[cfg(not(any(windows, target_os = "redox")))] @@ -345,88 +304,102 @@ fn get_all_filesystems(opt: &Options) -> Result, std::io::Error> } } - // The list of all mounted filesystems. - // - // Filesystems excluded by the command-line options are - // not considered. - let mounts: Vec = filter_mount_list(read_fs_list()?, opt); + let mut mounts = vec![]; + for mi in read_fs_list()? { + // TODO The running time of the `is_best()` function is linear + // in the length of `result`. That makes the running time of + // this loop quadratic in the length of `vmi`. This could be + // improved by a more efficient implementation of `is_best()`, + // but `vmi` is probably not very long in practice. + if is_included(&mi, opt) && is_best(&mounts, &mi) { + mounts.push(mi); + } + } // Convert each `MountInfo` into a `Filesystem`, which contains // both the mount information and usage information. - Ok(mounts - .into_iter() - .filter_map(|m| Filesystem::new(m, None)) - .filter(|fs| opt.show_all_fs || fs.usage.blocks > 0) - .collect()) + #[cfg(not(windows))] + { + let maybe_mount = |m| Filesystem::from_mount(&mounts, &m, None).ok(); + Ok(mounts + .clone() + .into_iter() + .filter_map(maybe_mount) + .filter(|fs| opt.show_all_fs || fs.usage.blocks > 0) + .collect()) + } + #[cfg(windows)] + { + let maybe_mount = |m| Filesystem::from_mount(&m, None).ok(); + Ok(mounts + .into_iter() + .filter_map(maybe_mount) + .filter(|fs| opt.show_all_fs || fs.usage.blocks > 0) + .collect()) + } } /// For each path, get the filesystem that contains that path. -fn get_named_filesystems

(paths: &[P], opt: &Options) -> Result, std::io::Error> +fn get_named_filesystems

(paths: &[P], opt: &Options) -> UResult> where P: AsRef, { // The list of all mounted filesystems. - // - // Filesystems marked as `dummy` or of type "lofs" are not - // considered. The "lofs" filesystem is a loopback - // filesystem present on Solaris and FreeBSD systems. It - // is similar to a symbolic link. - let mounts: Vec = filter_mount_list(read_fs_list()?, opt) - .into_iter() - .filter(|mi| mi.fs_type != "lofs" && !mi.dummy) - .collect(); + let mounts: Vec = read_fs_list()?; let mut result = vec![]; - // this happens if the file system type doesn't exist - if mounts.is_empty() { - show!(USimpleError::new(1, "no file systems processed")); - return Ok(result); - } - // Convert each path into a `Filesystem`, which contains // both the mount information and usage information. for path in paths { match Filesystem::from_path(&mounts, path) { - Some(fs) => result.push(fs), - None => { - // this happens if specified file system type != file system type of the file - if path.as_ref().exists() { - show!(USimpleError::new(1, "no file systems processed")); - } else { - show!(USimpleError::new( - 1, - format!("{}: No such file or directory", path.as_ref().display()) - )); + Ok(fs) => { + if is_included(&fs.mount_info, opt) { + result.push(fs); } } + Err(FsError::InvalidPath) => { + show!(USimpleError::new( + 1, + format!("{}: No such file or directory", path.as_ref().display()) + )); + } + Err(FsError::MountMissing) => { + show!(USimpleError::new(1, "no file systems processed")); + } + #[cfg(not(windows))] + Err(FsError::OverMounted) => { + show!(USimpleError::new( + 1, + format!( + "cannot access {}: over-mounted by another device", + path.as_ref().quote() + ) + )); + } } } + if get_exit_code() == 0 && result.is_empty() { + show!(USimpleError::new(1, "no file systems processed")); + return Ok(result); + } + Ok(result) } -#[derive(Debug)] +#[derive(Debug, Error)] enum DfError { /// A problem while parsing command-line options. + #[error("{}", .0)] OptionsError(OptionsError), } -impl Error for DfError {} - impl UError for DfError { fn usage(&self) -> bool { matches!(self, Self::OptionsError(OptionsError::ColumnError(_))) } } -impl fmt::Display for DfError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::OptionsError(e) => e.fmt(f), - } - } -} - #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let matches = uu_app().try_get_matches_from(args)?; @@ -443,8 +416,10 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { // Get the list of filesystems to display in the output table. let filesystems: Vec = match matches.get_many::(OPT_PATHS) { None => { - let filesystems = get_all_filesystems(&opt) - .map_err_context(|| "cannot read table of mounted file systems".into())?; + let filesystems = get_all_filesystems(&opt).map_err(|e| { + let context = "cannot read table of mounted file systems"; + USimpleError::new(e.code(), format!("{context}: {e}")) + })?; if filesystems.is_empty() { return Err(USimpleError::new(1, "no file systems processed")); @@ -454,8 +429,10 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } Some(paths) => { let paths: Vec<_> = paths.collect(); - let filesystems = get_named_filesystems(&paths, &opt) - .map_err_context(|| "cannot read table of mounted file systems".into())?; + let filesystems = get_named_filesystems(&paths, &opt).map_err(|e| { + let context = "cannot read table of mounted file systems"; + USimpleError::new(e.code(), format!("{context}: {e}")) + })?; // This can happen if paths are given as command-line arguments // but none of the paths exist. @@ -474,7 +451,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .after_help(AFTER_HELP) @@ -724,7 +701,7 @@ mod tests { mod is_included { - use crate::{is_included, Options}; + use crate::{Options, is_included}; use uucore::fsext::MountInfo; /// Instantiate a [`MountInfo`] with the given fields. @@ -858,16 +835,4 @@ mod tests { assert!(is_included(&m, &opt)); } } - - mod filter_mount_list { - - use crate::{filter_mount_list, Options}; - - #[test] - fn test_empty() { - let opt = Options::default(); - let mount_infos = vec![]; - assert!(filter_mount_list(mount_infos, &opt).is_empty()); - } - } } diff --git a/src/uu/df/src/filesystem.rs b/src/uu/df/src/filesystem.rs index ef5107958a2..43b1deb36c2 100644 --- a/src/uu/df/src/filesystem.rs +++ b/src/uu/df/src/filesystem.rs @@ -37,6 +37,33 @@ pub(crate) struct Filesystem { pub usage: FsUsage, } +#[derive(Debug, PartialEq)] +pub(crate) enum FsError { + #[cfg(not(windows))] + OverMounted, + InvalidPath, + MountMissing, +} + +/// Check whether `mount` has been over-mounted. +/// +/// `mount` is considered over-mounted if it there is an element in +/// `mounts` after mount that has the same mount_dir. +#[cfg(not(windows))] +fn is_over_mounted(mounts: &[MountInfo], mount: &MountInfo) -> bool { + let last_mount_for_dir = mounts + .iter() + .filter(|m| m.mount_dir == mount.mount_dir) + .next_back(); + + if let Some(lmi) = last_mount_for_dir { + lmi.dev_name != mount.dev_name + } else { + // Should be unreachable if `mount` is in `mounts` + false + } +} + /// Find the mount info that best matches a given filesystem path. /// /// This function returns the element of `mounts` on which `path` is @@ -56,14 +83,16 @@ fn mount_info_from_path

( path: P, // This is really only used for testing purposes. canonicalize: bool, -) -> Option<&MountInfo> +) -> Result<&MountInfo, FsError> where P: AsRef, { // TODO Refactor this function with `Stater::find_mount_point()` // in the `stat` crate. let path = if canonicalize { - path.as_ref().canonicalize().ok()? + path.as_ref() + .canonicalize() + .map_err(|_| FsError::InvalidPath)? } else { path.as_ref().to_path_buf() }; @@ -82,12 +111,14 @@ where .find(|m| m.1.eq(&path)) .map(|m| m.0); - maybe_mount_point.or_else(|| { - mounts - .iter() - .filter(|mi| path.starts_with(&mi.mount_dir)) - .max_by_key(|mi| mi.mount_dir.len()) - }) + maybe_mount_point + .or_else(|| { + mounts + .iter() + .filter(|mi| path.starts_with(&mi.mount_dir)) + .max_by_key(|mi| mi.mount_dir.len()) + }) + .ok_or(FsError::MountMissing) } impl Filesystem { @@ -109,14 +140,35 @@ impl Filesystem { #[cfg(unix)] let usage = FsUsage::new(statfs(_stat_path).ok()?); #[cfg(windows)] - let usage = FsUsage::new(Path::new(&_stat_path)); + let usage = FsUsage::new(Path::new(&_stat_path)).ok()?; Some(Self { + file, mount_info, usage, - file, }) } + /// Find and create the filesystem from the given mount + /// after checking that the it hasn't been over-mounted + #[cfg(not(windows))] + pub(crate) fn from_mount( + mounts: &[MountInfo], + mount: &MountInfo, + file: Option, + ) -> Result { + if is_over_mounted(mounts, mount) { + Err(FsError::OverMounted) + } else { + Self::new(mount.clone(), file).ok_or(FsError::MountMissing) + } + } + + /// Find and create the filesystem from the given mount. + #[cfg(windows)] + pub(crate) fn from_mount(mount: &MountInfo, file: Option) -> Result { + Self::new(mount.clone(), file).ok_or(FsError::MountMissing) + } + /// Find and create the filesystem that best matches a given path. /// /// This function returns a new `Filesystem` derived from the @@ -133,16 +185,18 @@ impl Filesystem { /// * [`Path::canonicalize`] /// * [`MountInfo::mount_dir`] /// - pub(crate) fn from_path

(mounts: &[MountInfo], path: P) -> Option + pub(crate) fn from_path

(mounts: &[MountInfo], path: P) -> Result where P: AsRef, { let file = path.as_ref().display().to_string(); let canonicalize = true; - let mount_info = mount_info_from_path(mounts, path, canonicalize)?; - // TODO Make it so that we do not need to clone the `mount_info`. - let mount_info = (*mount_info).clone(); - Self::new(mount_info, Some(file)) + + let result = mount_info_from_path(mounts, path, canonicalize); + #[cfg(windows)] + return result.and_then(|mount_info| Self::from_mount(mount_info, Some(file))); + #[cfg(not(windows))] + return result.and_then(|mount_info| Self::from_mount(mounts, mount_info, Some(file))); } } @@ -153,7 +207,7 @@ mod tests { use uucore::fsext::MountInfo; - use crate::filesystem::mount_info_from_path; + use crate::filesystem::{FsError, mount_info_from_path}; // Create a fake `MountInfo` with the given directory name. fn mount_info(mount_dir: &str) -> MountInfo { @@ -183,7 +237,19 @@ mod tests { #[test] fn test_empty_mounts() { - assert!(mount_info_from_path(&[], "/", false).is_none()); + assert_eq!( + mount_info_from_path(&[], "/", false).unwrap_err(), + FsError::MountMissing + ); + } + + #[test] + fn test_bad_path() { + assert_eq!( + // This path better not exist.... + mount_info_from_path(&[], "/non-existent-path", true).unwrap_err(), + FsError::InvalidPath + ); } #[test] @@ -210,16 +276,25 @@ mod tests { #[test] fn test_no_match() { let mounts = [mount_info("/foo")]; - assert!(mount_info_from_path(&mounts, "/bar", false).is_none()); + assert_eq!( + mount_info_from_path(&mounts, "/bar", false).unwrap_err(), + FsError::MountMissing + ); } #[test] fn test_partial_match() { let mounts = [mount_info("/foo/bar")]; - assert!(mount_info_from_path(&mounts, "/foo/baz", false).is_none()); + assert_eq!( + mount_info_from_path(&mounts, "/foo/baz", false).unwrap_err(), + FsError::MountMissing + ); } #[test] + // clippy::assigning_clones added with Rust 1.78 + // Rust version = 1.76 on OpenBSD stable/7.5 + #[cfg_attr(not(target_os = "openbsd"), allow(clippy::assigning_clones))] fn test_dev_name_match() { let tmp = tempfile::TempDir::new().expect("Failed to create temp dir"); let dev_name = std::fs::canonicalize(tmp.path()) @@ -234,4 +309,52 @@ mod tests { assert!(mount_info_eq(actual, &mounts[0])); } } + + #[cfg(not(windows))] + mod over_mount { + use crate::filesystem::{Filesystem, FsError, is_over_mounted}; + use uucore::fsext::MountInfo; + + fn mount_info_with_dev_name(mount_dir: &str, dev_name: Option<&str>) -> MountInfo { + MountInfo { + dev_id: String::default(), + dev_name: dev_name.map(String::from).unwrap_or_default(), + fs_type: String::default(), + mount_dir: String::from(mount_dir), + mount_option: String::default(), + mount_root: String::default(), + remote: Default::default(), + dummy: Default::default(), + } + } + + #[test] + fn test_over_mount() { + let mount_info1 = mount_info_with_dev_name("/foo", Some("dev_name_1")); + let mount_info2 = mount_info_with_dev_name("/foo", Some("dev_name_2")); + let mounts = [mount_info1, mount_info2]; + assert!(is_over_mounted(&mounts, &mounts[0])); + } + + #[test] + fn test_over_mount_not_over_mounted() { + let mount_info1 = mount_info_with_dev_name("/foo", Some("dev_name_1")); + let mount_info2 = mount_info_with_dev_name("/foo", Some("dev_name_2")); + let mounts = [mount_info1, mount_info2]; + assert!(!is_over_mounted(&mounts, &mounts[1])); + } + + #[test] + fn test_from_mount_over_mounted() { + let mount_info1 = mount_info_with_dev_name("/foo", Some("dev_name_1")); + let mount_info2 = mount_info_with_dev_name("/foo", Some("dev_name_2")); + + let mounts = [mount_info1, mount_info2]; + + assert_eq!( + Filesystem::from_mount(&mounts, &mounts[0], None).unwrap_err(), + FsError::OverMounted + ); + } + } } diff --git a/src/uu/df/src/table.rs b/src/uu/df/src/table.rs index f6e09420482..be7eb8557f9 100644 --- a/src/uu/df/src/table.rs +++ b/src/uu/df/src/table.rs @@ -9,7 +9,7 @@ //! collection of data rows ([`Row`]), one per filesystem. use unicode_width::UnicodeWidthStr; -use crate::blocks::{to_magnitude_and_suffix, SuffixType}; +use crate::blocks::{SuffixType, to_magnitude_and_suffix}; use crate::columns::{Alignment, Column}; use crate::filesystem::Filesystem; use crate::{BlockSize, Options}; @@ -58,13 +58,13 @@ pub(crate) struct Row { bytes_capacity: Option, /// Total number of inodes in the filesystem. - inodes: u64, + inodes: u128, /// Number of used inodes. - inodes_used: u64, + inodes_used: u128, /// Number of free inodes. - inodes_free: u64, + inodes_free: u128, /// Percentage of inodes that are used, given as a float between 0 and 1. /// @@ -178,9 +178,9 @@ impl From for Row { } else { Some(bavail as f64 / ((bused + bavail) as f64)) }, - inodes: files, - inodes_used: fused, - inodes_free: ffree, + inodes: files as u128, + inodes_used: fused as u128, + inodes_free: ffree as u128, inodes_usage: if files == 0 { None } else { @@ -235,9 +235,9 @@ impl<'a> RowFormatter<'a> { /// Get a string giving the scaled version of the input number. /// /// The scaling factor is defined in the `options` field. - fn scaled_inodes(&self, size: u64) -> String { + fn scaled_inodes(&self, size: u128) -> String { if let Some(h) = self.options.human_readable { - to_magnitude_and_suffix(size.into(), SuffixType::HumanReadable(h)) + to_magnitude_and_suffix(size, SuffixType::HumanReadable(h)) } else { size.to_string() } @@ -395,12 +395,6 @@ impl Table { let values = fmt.get_values(); total += row; - for (i, value) in values.iter().enumerate() { - if UnicodeWidthStr::width(value.as_str()) > widths[i] { - widths[i] = UnicodeWidthStr::width(value.as_str()); - } - } - rows.push(values); } } @@ -410,6 +404,16 @@ impl Table { rows.push(total_row.get_values()); } + // extend the column widths (in chars) for long values in rows + // do it here, after total row was added to the list of rows + for row in &rows { + for (i, value) in row.iter().enumerate() { + if UnicodeWidthStr::width(value.as_str()) > widths[i] { + widths[i] = UnicodeWidthStr::width(value.as_str()); + } + } + } + Self { rows, widths, @@ -466,9 +470,11 @@ impl fmt::Display for Table { #[cfg(test)] mod tests { + use std::vec; + use crate::blocks::HumanReadable; use crate::columns::Column; - use crate::table::{Header, HeaderMode, Row, RowFormatter}; + use crate::table::{Header, HeaderMode, Row, RowFormatter, Table}; use crate::{BlockSize, Options}; const COLUMNS_WITH_FS_TYPE: [Column; 7] = [ @@ -835,12 +841,12 @@ mod tests { }, usage: crate::table::FsUsage { blocksize: 4096, - blocks: 244029695, - bfree: 125085030, - bavail: 125085030, + blocks: 244_029_695, + bfree: 125_085_030, + bavail: 125_085_030, bavail_top_bit_set: false, files: 999, - ffree: 1000000, + ffree: 1_000_000, }, }; @@ -848,4 +854,89 @@ mod tests { assert_eq!(row.inodes_used, 0); } + + #[test] + fn test_table_column_width_computation_include_total_row() { + let d1 = crate::Filesystem { + file: None, + mount_info: crate::MountInfo { + dev_id: "28".to_string(), + dev_name: "none".to_string(), + fs_type: "9p".to_string(), + mount_dir: "/usr/lib/wsl/drivers".to_string(), + mount_option: "ro,nosuid,nodev,noatime".to_string(), + mount_root: "/".to_string(), + remote: false, + dummy: false, + }, + usage: crate::table::FsUsage { + blocksize: 4096, + blocks: 244_029_695, + bfree: 125_085_030, + bavail: 125_085_030, + bavail_top_bit_set: false, + files: 99_999_999_999, + ffree: 999_999, + }, + }; + + let filesystems = vec![d1.clone(), d1]; + + let mut options = Options { + show_total: true, + columns: vec![ + Column::Source, + Column::Itotal, + Column::Iused, + Column::Iavail, + ], + ..Default::default() + }; + + let table_w_total = Table::new(&options, filesystems.clone()); + assert_eq!( + table_w_total.to_string(), + "Filesystem Inodes IUsed IFree\n\ + none 99999999999 99999000000 999999\n\ + none 99999999999 99999000000 999999\n\ + total 199999999998 199998000000 1999998" + ); + + options.show_total = false; + + let table_w_o_total = Table::new(&options, filesystems); + assert_eq!( + table_w_o_total.to_string(), + "Filesystem Inodes IUsed IFree\n\ + none 99999999999 99999000000 999999\n\ + none 99999999999 99999000000 999999" + ); + } + + #[test] + fn test_row_accumulation_u64_overflow() { + let total = u64::MAX as u128; + let used1 = 3000u128; + let used2 = 50000u128; + + let mut row1 = Row { + inodes: total, + inodes_used: used1, + inodes_free: total - used1, + ..Default::default() + }; + + let row2 = Row { + inodes: total, + inodes_used: used2, + inodes_free: total - used2, + ..Default::default() + }; + + row1 += row2; + + assert_eq!(row1.inodes, total * 2); + assert_eq!(row1.inodes_used, used1 + used2); + assert_eq!(row1.inodes_free, total * 2 - used1 - used2); + } } diff --git a/src/uu/dir/Cargo.toml b/src/uu/dir/Cargo.toml index b82298ce319..9bbec793b40 100644 --- a/src/uu/dir/Cargo.toml +++ b/src/uu/dir/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_dir" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "shortcut to ls -C -b" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/ls" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/dir.rs" diff --git a/src/uu/dir/src/dir.rs b/src/uu/dir/src/dir.rs index e255295119a..0a8a71c228a 100644 --- a/src/uu/dir/src/dir.rs +++ b/src/uu/dir/src/dir.rs @@ -6,7 +6,7 @@ use clap::Command; use std::ffi::OsString; use std::path::Path; -use uu_ls::{options, Config, Format}; +use uu_ls::{Config, Format, options}; use uucore::error::UResult; use uucore::quoting_style::{Quotes, QuotingStyle}; diff --git a/src/uu/dircolors/Cargo.toml b/src/uu/dircolors/Cargo.toml index 1bf87c22cc1..5403cd1b40f 100644 --- a/src/uu/dircolors/Cargo.toml +++ b/src/uu/dircolors/Cargo.toml @@ -1,22 +1,25 @@ [package] name = "uu_dircolors" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "dircolors ~ (uutils) display commands to set LS_COLORS" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/dircolors" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/dircolors.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true, features = ["colors"] } +uucore = { workspace = true, features = ["colors", "parser"] } [[bin]] name = "dircolors" diff --git a/src/uu/dircolors/README.md b/src/uu/dircolors/README.md index ce8aa965f93..62944d4907b 100644 --- a/src/uu/dircolors/README.md +++ b/src/uu/dircolors/README.md @@ -15,4 +15,4 @@ Run the tests: cargo test --features "dircolors" --no-default-features ``` -Edit `/PATH_TO_COREUTILS/src/uu/dircolors/src/colors.rs` until the tests pass. +Edit `/PATH_TO_COREUTILS/src/uu/dircolors/src/dircolors.rs` until the tests pass. diff --git a/src/uu/dircolors/src/dircolors.rs b/src/uu/dircolors/src/dircolors.rs index 531c3ee474c..4fb9228eb5f 100644 --- a/src/uu/dircolors/src/dircolors.rs +++ b/src/uu/dircolors/src/dircolors.rs @@ -3,20 +3,20 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) clrtoeol dircolors eightbit endcode fnmatch leftcode multihardlink rightcode setenv sgid suid colorterm +// spell-checker:ignore (ToDO) clrtoeol dircolors eightbit endcode fnmatch leftcode multihardlink rightcode setenv sgid suid colorterm disp use std::borrow::Borrow; use std::env; +use std::fmt::Write as _; use std::fs::File; -//use std::io::IsTerminal; use std::io::{BufRead, BufReader}; use std::path::Path; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use uucore::colors::{FILE_ATTRIBUTE_CODES, FILE_COLORS, FILE_TYPES, TERMS}; use uucore::display::Quotable; use uucore::error::{UResult, USimpleError, UUsageError}; -use uucore::{help_about, help_section, help_usage}; +use uucore::{format_usage, help_about, help_section, help_usage, parser::parse_glob}; mod options { pub const BOURNE_SHELL: &str = "bourne-shell"; @@ -78,14 +78,14 @@ pub fn generate_type_output(fmt: &OutputFmt) -> String { match fmt { OutputFmt::Display => FILE_TYPES .iter() - .map(|&(_, key, val)| format!("\x1b[{}m{}\t{}\x1b[0m", val, key, val)) + .map(|&(_, key, val)| format!("\x1b[{val}m{key}\t{val}\x1b[0m")) .collect::>() .join("\n"), _ => { // Existing logic for other formats FILE_TYPES .iter() - .map(|&(_, v1, v2)| format!("{}={}", v1, v2)) + .map(|&(_, v1, v2)| format!("{v1}={v2}")) .collect::>() .join(":") } @@ -100,8 +100,7 @@ fn generate_ls_colors(fmt: &OutputFmt, sep: &str) -> String { display_parts.push(type_output); for &(extension, code) in FILE_COLORS { let prefix = if extension.starts_with('*') { "" } else { "*" }; - let formatted_extension = - format!("\x1b[{}m{}{}\t{}\x1b[0m", code, prefix, extension, code); + let formatted_extension = format!("\x1b[{code}m{prefix}{extension}\t{code}\x1b[0m"); display_parts.push(formatted_extension); } display_parts.join("\n") @@ -111,18 +110,12 @@ fn generate_ls_colors(fmt: &OutputFmt, sep: &str) -> String { let mut parts = vec![]; for &(extension, code) in FILE_COLORS { let prefix = if extension.starts_with('*') { "" } else { "*" }; - let formatted_extension = format!("{}{}", prefix, extension); - parts.push(format!("{}={}", formatted_extension, code)); + let formatted_extension = format!("{prefix}{extension}"); + parts.push(format!("{formatted_extension}={code}")); } let (prefix, suffix) = get_colors_format_strings(fmt); let ls_colors = parts.join(sep); - format!( - "{}{}:{}:{}", - prefix, - generate_type_output(fmt), - ls_colors, - suffix - ) + format!("{prefix}{}:{ls_colors}:{suffix}", generate_type_output(fmt),) } } } @@ -235,10 +228,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { ); } Err(e) => { - return Err(USimpleError::new( - 1, - format!("{}: {}", path.maybe_quote(), e), - )); + return Err(USimpleError::new(1, format!("{}: {e}", path.maybe_quote()))); } } } @@ -254,10 +244,11 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .after_help(AFTER_HELP) .override_usage(format_usage(USAGE)) + .args_override_self(true) .infer_long_args(true) .arg( Arg::new(options::BOURNE_SHELL) @@ -359,9 +350,6 @@ enum ParseState { Pass, } -use uucore::{format_usage, parse_glob}; - -#[allow(clippy::cognitive_complexity)] fn parse(user_input: T, fmt: &OutputFmt, fp: &str) -> Result where T: IntoIterator, @@ -372,10 +360,12 @@ where result.push_str(&prefix); + // Get environment variables once at the start let term = env::var("TERM").unwrap_or_else(|_| "none".to_owned()); - let term = term.as_str(); + let colorterm = env::var("COLORTERM").unwrap_or_default(); let mut state = ParseState::Global; + let mut saw_colorterm_match = false; for (num, line) in user_input.into_iter().enumerate() { let num = num + 1; @@ -390,57 +380,42 @@ where if val.is_empty() { return Err(format!( // The double space is what GNU is doing - "{}:{}: invalid line; missing second token", + "{}:{num}: invalid line; missing second token", fp.maybe_quote(), - num )); } + let lower = key.to_lowercase(); - if lower == "term" || lower == "colorterm" { - if term.fnmatch(val) { - state = ParseState::Matched; - } else if state != ParseState::Matched { - state = ParseState::Pass; - } - } else { - if state == ParseState::Matched { - // prevent subsequent mismatched TERM from - // cancelling the input - state = ParseState::Continue; + match lower.as_str() { + "term" => { + if term.fnmatch(val) { + state = ParseState::Matched; + } else if state == ParseState::Global { + state = ParseState::Pass; + } } - if state != ParseState::Pass { - let search_key = lower.as_str(); - - if key.starts_with('.') { - if *fmt == OutputFmt::Display { - result.push_str(format!("\x1b[{val}m*{key}\t{val}\x1b[0m\n").as_str()); - } else { - result.push_str(format!("*{key}={val}:").as_str()); - } - } else if key.starts_with('*') { - if *fmt == OutputFmt::Display { - result.push_str(format!("\x1b[{val}m{key}\t{val}\x1b[0m\n").as_str()); - } else { - result.push_str(format!("{key}={val}:").as_str()); - } - } else if lower == "options" || lower == "color" || lower == "eightbit" { - // Slackware only. Ignore - } else if let Some((_, s)) = FILE_ATTRIBUTE_CODES - .iter() - .find(|&&(key, _)| key == search_key) - { - if *fmt == OutputFmt::Display { - result.push_str(format!("\x1b[{val}m{s}\t{val}\x1b[0m\n").as_str()); - } else { - result.push_str(format!("{s}={val}:").as_str()); - } + "colorterm" => { + // For COLORTERM ?*, only match if COLORTERM is non-empty + let matches = if val == "?*" { + !colorterm.is_empty() } else { - return Err(format!( - "{}:{}: unrecognized keyword {}", - fp.maybe_quote(), - num, - key - )); + colorterm.fnmatch(val) + }; + if matches { + state = ParseState::Matched; + saw_colorterm_match = true; + } else if !saw_colorterm_match && state == ParseState::Global { + state = ParseState::Pass; + } + } + _ => { + if state == ParseState::Matched { + // prevent subsequent mismatched TERM from + // cancelling the input + state = ParseState::Continue; + } + if state != ParseState::Pass { + append_entry(&mut result, fmt, key, &lower, val)?; } } } @@ -455,6 +430,46 @@ where Ok(result) } +fn append_entry( + result: &mut String, + fmt: &OutputFmt, + key: &str, + lower: &str, + val: &str, +) -> Result<(), String> { + if key.starts_with(['.', '*']) { + let entry = if key.starts_with('.') { + format!("*{key}") + } else { + key.to_string() + }; + let disp = if *fmt == OutputFmt::Display { + format!("\x1b[{val}m{entry}\t{val}\x1b[0m\n") + } else { + format!("{entry}={val}:") + }; + result.push_str(&disp); + return Ok(()); + } + + match lower { + "options" | "color" | "eightbit" => Ok(()), // Slackware only, ignore + _ => { + if let Some((_, s)) = FILE_ATTRIBUTE_CODES.iter().find(|&&(key, _)| key == lower) { + let disp = if *fmt == OutputFmt::Display { + format!("\x1b[{val}m{s}\t{val}\x1b[0m\n") + } else { + format!("{s}={val}:") + }; + result.push_str(&disp); + Ok(()) + } else { + Err(format!("unrecognized keyword {key}")) + } + } + } +} + /// Escape single quotes because they are not allowed between single quotes in shell code, and code /// enclosed by single quotes is what is returned by `parse()`. /// @@ -468,7 +483,7 @@ fn escape(s: &str) -> String { match c { '\'' => result.push_str("'\\''"), ':' if previous != '\\' => result.push_str("\\:"), - _ => result.push_str(&c.to_string()), + _ => result.push(c), } previous = c; } @@ -492,7 +507,7 @@ pub fn generate_dircolors_config() -> String { ); config.push_str("COLORTERM ?*\n"); for term in TERMS { - config.push_str(&format!("TERM {}\n", term)); + let _ = writeln!(config, "TERM {term}"); } config.push_str( @@ -513,14 +528,14 @@ pub fn generate_dircolors_config() -> String { ); for (name, _, code) in FILE_TYPES { - config.push_str(&format!("{} {}\n", name, code)); + let _ = writeln!(config, "{name} {code}"); } config.push_str("# List any file extensions like '.gz' or '.tar' that you would like ls\n"); config.push_str("# to color below. Put the extension, a space, and the color init string.\n"); for (ext, color) in FILE_COLORS { - config.push_str(&format!("{} {}\n", ext, color)); + let _ = writeln!(config, "{ext} {color}"); } config.push_str("# Subsequent TERM or COLORTERM entries, can be used to add / override\n"); config.push_str("# config specific to those matching environment variables."); diff --git a/src/uu/dirname/Cargo.toml b/src/uu/dirname/Cargo.toml index 85391859663..7e505a37b9f 100644 --- a/src/uu/dirname/Cargo.toml +++ b/src/uu/dirname/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_dirname" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "dirname ~ (uutils) display parent directory of PATHNAME" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/dirname" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/dirname.rs" diff --git a/src/uu/dirname/src/dirname.rs b/src/uu/dirname/src/dirname.rs index a645b05fd72..de8740f8970 100644 --- a/src/uu/dirname/src/dirname.rs +++ b/src/uu/dirname/src/dirname.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use std::path::Path; use uucore::display::print_verbatim; use uucore::error::{UResult, UUsageError}; @@ -62,8 +62,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) .about(ABOUT) - .version(crate_version!()) + .version(uucore::crate_version!()) .override_usage(format_usage(USAGE)) + .args_override_self(true) .infer_long_args(true) .arg( Arg::new(options::ZERO) diff --git a/src/uu/du/Cargo.toml b/src/uu/du/Cargo.toml index 8b9eb062e48..5b0d3f5e8ea 100644 --- a/src/uu/du/Cargo.toml +++ b/src/uu/du/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_du" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "du ~ (uutils) display disk usage" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/du" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/du.rs" @@ -19,7 +22,8 @@ chrono = { workspace = true } # For the --exclude & --exclude-from options glob = { workspace = true } clap = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["format", "parser"] } +thiserror = { workspace = true } [target.'cfg(target_os = "windows")'.dependencies] windows-sys = { workspace = true, features = [ diff --git a/src/uu/du/src/du.rs b/src/uu/du/src/du.rs index 62fcfceda01..0b268888136 100644 --- a/src/uu/du/src/du.rs +++ b/src/uu/du/src/du.rs @@ -4,15 +4,13 @@ // file that was distributed with this source code. use chrono::{DateTime, Local}; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command, builder::PossibleValue}; use glob::Pattern; use std::collections::HashSet; use std::env; -use std::error::Error; -use std::fmt::Display; #[cfg(not(windows))] use std::fs::Metadata; -use std::fs::{self, File}; +use std::fs::{self, DirEntry, File}; use std::io::{BufRead, BufReader}; #[cfg(not(windows))] use std::os::unix::fs::MetadataExt; @@ -25,18 +23,20 @@ use std::str::FromStr; use std::sync::mpsc; use std::thread; use std::time::{Duration, UNIX_EPOCH}; -use uucore::display::{print_verbatim, Quotable}; -use uucore::error::{set_exit_code, FromIo, UError, UResult, USimpleError}; +use thiserror::Error; +use uucore::display::{Quotable, print_verbatim}; +use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code}; use uucore::line_ending::LineEnding; -use uucore::parse_glob; -use uucore::parse_size::{parse_size_u64, ParseSizeError}; +use uucore::parser::parse_glob; +use uucore::parser::parse_size::{ParseSizeError, parse_size_u64}; +use uucore::parser::shortcut_value_parser::ShortcutValueParser; use uucore::{format_usage, help_about, help_section, help_usage, show, show_error, show_warning}; #[cfg(windows)] use windows_sys::Win32::Foundation::HANDLE; #[cfg(windows)] use windows_sys::Win32::Storage::FileSystem::{ - FileIdInfo, FileStandardInfo, GetFileInformationByHandleEx, FILE_ID_128, FILE_ID_INFO, - FILE_STANDARD_INFO, + FILE_ID_128, FILE_ID_INFO, FILE_STANDARD_INFO, FileIdInfo, FileStandardInfo, + GetFileInformationByHandleEx, }; mod options { @@ -74,9 +74,6 @@ const ABOUT: &str = help_about!("du.md"); const AFTER_HELP: &str = help_section!("after help", "du.md"); const USAGE: &str = help_usage!("du.md"); -// TODO: Support Z & Y (currently limited by size of u64) -const UNITS: [(char, u32); 6] = [('E', 6), ('P', 5), ('T', 4), ('G', 3), ('M', 2), ('K', 1)]; - struct TraversalOptions { all: bool, separate_dirs: bool, @@ -116,7 +113,8 @@ enum Time { #[derive(Clone)] enum SizeFormat { - Human(u64), + HumanDecimal, + HumanBinary, BlockSize(u64), } @@ -139,7 +137,11 @@ struct Stat { } impl Stat { - fn new(path: &Path, options: &TraversalOptions) -> std::io::Result { + fn new( + path: &Path, + dir_entry: Option<&DirEntry>, + options: &TraversalOptions, + ) -> std::io::Result { // Determine whether to dereference (follow) the symbolic link let should_dereference = match &options.dereference { Deref::All => true, @@ -150,8 +152,11 @@ impl Stat { let metadata = if should_dereference { // Get metadata, following symbolic links if necessary fs::metadata(path) + } else if let Some(dir_entry) = dir_entry { + // Get metadata directly from the DirEntry, which is faster on Windows + dir_entry.metadata() } else { - // Get metadata without following symbolic links + // Get metadata from the filesystem without following symbolic links fs::symlink_metadata(path) }?; @@ -165,7 +170,7 @@ impl Stat { Ok(Self { path: path.to_path_buf(), is_dir: metadata.is_dir(), - size: if path.is_dir() { 0 } else { metadata.len() }, + size: if metadata.is_dir() { 0 } else { metadata.len() }, blocks: metadata.blocks(), inodes: 1, inode: Some(file_info), @@ -183,7 +188,7 @@ impl Stat { Ok(Self { path: path.to_path_buf(), is_dir: metadata.is_dir(), - size: if path.is_dir() { 0 } else { metadata.len() }, + size: if metadata.is_dir() { 0 } else { metadata.len() }, blocks: size_on_disk / 1024 * 2, inodes: 1, inode: file_info, @@ -222,9 +227,8 @@ fn get_size_on_disk(path: &Path) -> u64 { // bind file so it stays in scope until end of function // if it goes out of scope the handle below becomes invalid - let file = match fs::File::open(path) { - Ok(file) => file, - Err(_) => return size_on_disk, // opening directories will fail + let Ok(file) = File::open(path) else { + return size_on_disk; // opening directories will fail }; unsafe { @@ -234,8 +238,8 @@ fn get_size_on_disk(path: &Path) -> u64 { let success = GetFileInformationByHandleEx( file.as_raw_handle() as HANDLE, FileStandardInfo, - file_info_ptr as _, - std::mem::size_of::() as u32, + file_info_ptr.cast(), + size_of::() as u32, ); if success != 0 { @@ -250,9 +254,8 @@ fn get_size_on_disk(path: &Path) -> u64 { fn get_file_info(path: &Path) -> Option { let mut result = None; - let file = match fs::File::open(path) { - Ok(file) => file, - Err(_) => return result, + let Ok(file) = File::open(path) else { + return result; }; unsafe { @@ -262,8 +265,8 @@ fn get_file_info(path: &Path) -> Option { let success = GetFileInformationByHandleEx( file.as_raw_handle() as HANDLE, FileIdInfo, - file_info_ptr as _, - std::mem::size_of::() as u32, + file_info_ptr.cast(), + size_of::() as u32, ); if success != 0 { @@ -320,7 +323,7 @@ fn du( 'file_loop: for f in read { match f { Ok(entry) => { - match Stat::new(&entry.path(), options) { + match Stat::new(&entry.path(), Some(&entry), options) { Ok(this_stat) => { // We have an exclude list for pattern in &options.excludes { @@ -340,14 +343,21 @@ fn du( } if let Some(inode) = this_stat.inode { - if seen_inodes.contains(&inode) { - if options.count_links { + // Check if the inode has been seen before and if we should skip it + if seen_inodes.contains(&inode) + && (!options.count_links || !options.all) + { + // If `count_links` is enabled and `all` is not, increment the inode count + if options.count_links && !options.all { my_stat.inodes += 1; } + // Skip further processing for this inode continue; } + // Mark this inode as seen seen_inodes.insert(inode); } + if this_stat.is_dir { if options.one_file_system { if let (Some(this_inode), Some(my_inode)) = @@ -396,48 +406,26 @@ fn du( Ok(my_stat) } -#[derive(Debug)] +#[derive(Debug, Error)] enum DuError { + #[error("invalid maximum depth {depth}", depth = .0.quote())] InvalidMaxDepthArg(String), + + #[error("summarizing conflicts with --max-depth={depth}", depth = .0.maybe_quote())] SummarizeDepthConflict(String), + + #[error("invalid argument {style} for 'time style'\nValid arguments are:\n- 'full-iso'\n- 'long-iso'\n- 'iso'\nTry '{help}' for more information.", + style = .0.quote(), + help = uucore::execution_phrase())] InvalidTimeStyleArg(String), + + #[error("'birth' and 'creation' arguments for --time are not supported on this platform.")] InvalidTimeArg, - InvalidGlob(String), -} -impl Display for DuError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::InvalidMaxDepthArg(s) => write!(f, "invalid maximum depth {}", s.quote()), - Self::SummarizeDepthConflict(s) => { - write!( - f, - "summarizing conflicts with --max-depth={}", - s.maybe_quote() - ) - } - Self::InvalidTimeStyleArg(s) => write!( - f, - "invalid argument {} for 'time style' -Valid arguments are: -- 'full-iso' -- 'long-iso' -- 'iso' -Try '{} --help' for more information.", - s.quote(), - uucore::execution_phrase() - ), - Self::InvalidTimeArg => write!( - f, - "'birth' and 'creation' arguments for --time are not supported on this platform.", - ), - Self::InvalidGlob(s) => write!(f, "Invalid exclude syntax: {s}"), - } - } + #[error("Invalid exclude syntax: {0}")] + InvalidGlob(String), } -impl Error for DuError {} - impl UError for DuError { fn code(&self) -> i32 { match self { @@ -476,7 +464,7 @@ fn build_exclude_patterns(matches: &ArgMatches) -> UResult> { let mut exclude_patterns = Vec::new(); for f in excludes_iterator.chain(exclude_from_iterator) { if matches.get_flag(options::VERBOSE) { - println!("adding {:?} to the exclude list ", &f); + println!("adding {f:?} to the exclude list "); } match parse_glob::from_str(&f) { Ok(glob) => exclude_patterns.push(glob), @@ -520,10 +508,10 @@ impl StatPrinter { if !self .threshold - .map_or(false, |threshold| threshold.should_exclude(size)) + .is_some_and(|threshold| threshold.should_exclude(size)) && self .max_depth - .map_or(true, |max_depth| stat_info.depth <= max_depth) + .is_none_or(|max_depth| stat_info.depth <= max_depth) && (!self.summarize || stat_info.depth == 0) { self.print_stat(&stat_info.stat, size)?; @@ -544,23 +532,23 @@ impl StatPrinter { } fn convert_size(&self, size: u64) -> String { - if self.inodes { - return size.to_string(); - } match self.size_format { - SizeFormat::Human(multiplier) => { - if size == 0 { - return "0".to_string(); - } - for &(unit, power) in &UNITS { - let limit = multiplier.pow(power); - if size >= limit { - return format!("{:.1}{}", (size as f64) / (limit as f64), unit); - } + SizeFormat::HumanDecimal => uucore::format::human::human_readable( + size, + uucore::format::human::SizeFormat::Decimal, + ), + SizeFormat::HumanBinary => uucore::format::human::human_readable( + size, + uucore::format::human::SizeFormat::Binary, + ), + SizeFormat::BlockSize(block_size) => { + if self.inodes { + // we ignore block size (-B) with --inodes + size.to_string() + } else { + size.div_ceil(block_size).to_string() } - format!("{size}B") } - SizeFormat::BlockSize(block_size) => div_ceil(size, block_size).to_string(), } } @@ -569,7 +557,7 @@ impl StatPrinter { let secs = get_time_secs(time, stat)?; let tm = DateTime::::from(UNIX_EPOCH + Duration::from_secs(secs)); let time_str = tm.format(&self.time_format).to_string(); - print!("{}\t{}\t", self.convert_size(size), time_str); + print!("{}\t{time_str}\t", self.convert_size(size)); } else { print!("{}\t", self.convert_size(size)); } @@ -581,13 +569,6 @@ impl StatPrinter { } } -// This can be replaced with u64::div_ceil once it is stabilized. -// This implementation approach is optimized for when `b` is a constant, -// particularly a power of two. -pub fn div_ceil(a: u64, b: u64) -> u64 { - (a + b - 1) / b -} - // Read file paths from the specified file, separated by null characters fn read_files_from(file_name: &str) -> Result, std::io::Error> { let reader: Box = if file_name == "-" { @@ -597,23 +578,18 @@ fn read_files_from(file_name: &str) -> Result, std::io::Error> { // First, check if the file_name is a directory let path = PathBuf::from(file_name); if path.is_dir() { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("{}: read error: Is a directory", file_name), - )); + return Err(std::io::Error::other(format!( + "{file_name}: read error: Is a directory" + ))); } // Attempt to open the file and handle the error if it does not exist match File::open(file_name) { Ok(file) => Box::new(BufReader::new(file)), Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!( - "cannot open '{}' for reading: No such file or directory", - file_name - ), - )) + return Err(std::io::Error::other(format!( + "cannot open '{file_name}' for reading: No such file or directory" + ))); } Err(e) => return Err(e), } @@ -646,6 +622,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let summarize = matches.get_flag(options::SUMMARIZE); + let count_links = matches.get_flag(options::COUNT_LINKS); + let max_depth = parse_depth( matches .get_one::(options::MAX_DEPTH) @@ -655,26 +633,27 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let files = if let Some(file_from) = matches.get_one::(options::FILES0_FROM) { if file_from == "-" && matches.get_one::(options::FILE).is_some() { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!( - "extra operand {}\nfile operands cannot be combined with --files0-from", - matches.get_one::(options::FILE).unwrap().quote() - ), - ) + return Err(std::io::Error::other(format!( + "extra operand {}\nfile operands cannot be combined with --files0-from", + matches.get_one::(options::FILE).unwrap().quote() + )) .into()); } read_files_from(file_from)? - } else { - match matches.get_one::(options::FILE) { - Some(_) => matches - .get_many::(options::FILE) - .unwrap() - .map(PathBuf::from) - .collect(), - None => vec![PathBuf::from(".")], + } else if let Some(files) = matches.get_many::(options::FILE) { + let files = files.map(PathBuf::from); + if count_links { + files.collect() + } else { + // Deduplicate while preserving order + let mut seen = HashSet::new(); + files + .filter(|path| seen.insert(path.clone())) + .collect::>() } + } else { + vec![PathBuf::from(".")] }; let time = matches.contains_id(options::TIME).then(|| { @@ -687,9 +666,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { }); let size_format = if matches.get_flag(options::HUMAN_READABLE) { - SizeFormat::Human(1024) + SizeFormat::HumanBinary } else if matches.get_flag(options::SI) { - SizeFormat::Human(1000) + SizeFormat::HumanDecimal } else if matches.get_flag(options::BYTES) { SizeFormat::BlockSize(1) } else if matches.get_flag(options::BLOCK_SIZE_1K) { @@ -697,11 +676,17 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } else if matches.get_flag(options::BLOCK_SIZE_1M) { SizeFormat::BlockSize(1024 * 1024) } else { - SizeFormat::BlockSize(read_block_size( - matches - .get_one::(options::BLOCK_SIZE) - .map(AsRef::as_ref), - )?) + let block_size_str = matches.get_one::(options::BLOCK_SIZE); + let block_size = read_block_size(block_size_str.map(AsRef::as_ref))?; + if block_size == 0 { + return Err(std::io::Error::other(format!( + "invalid --{} argument {}", + options::BLOCK_SIZE, + block_size_str.map_or("???BUG", |v| v).quote() + )) + .into()); + } + SizeFormat::BlockSize(block_size) }; let traversal_options = TraversalOptions { @@ -716,11 +701,17 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } else { Deref::None }, - count_links: matches.get_flag(options::COUNT_LINKS), + count_links, verbose: matches.get_flag(options::VERBOSE), excludes: build_exclude_patterns(&matches)?, }; + let time_format = if time.is_some() { + parse_time_style(matches.get_one::("time-style").map(|s| s.as_str()))?.to_string() + } else { + "%Y-%m-%d %H:%M".to_string() + }; + let stat_printer = StatPrinter { max_depth, size_format, @@ -737,8 +728,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { .transpose()?, apparent_size: matches.get_flag(options::APPARENT_SIZE) || matches.get_flag(options::BYTES), time, - time_format: parse_time_style(matches.get_one::("time-style").map(|s| s.as_str()))? - .to_string(), + time_format, line_ending: LineEnding::from_zero_flag(matches.get_flag(options::NULL)), }; @@ -768,7 +758,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } // Check existence of path provided in argument - if let Ok(stat) = Stat::new(&path, &traversal_options) { + if let Ok(stat) = Stat::new(&path, None, &traversal_options) { // Kick off the computation of disk usage from the initial path let mut seen_inodes: HashSet = HashSet::new(); if let Some(inode) = stat.inode { @@ -785,8 +775,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { .send(Err(USimpleError::new( 1, format!( - "{}: No such file or directory", - path.to_string_lossy().maybe_quote() + "cannot access {}: No such file or directory", + path.to_string_lossy().quote() ), ))) .map_err(|e| USimpleError::new(1, e.to_string()))?; @@ -833,7 +823,7 @@ fn parse_depth(max_depth_str: Option<&str>, summarize: bool) -> UResult Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .after_help(AFTER_HELP) .override_usage(format_usage(USAGE)) @@ -1035,7 +1025,11 @@ pub fn uu_app() -> Command { .value_name("WORD") .require_equals(true) .num_args(0..) - .value_parser(["atime", "access", "use", "ctime", "status", "birth", "creation"]) + .value_parser(ShortcutValueParser::new([ + PossibleValue::new("atime").alias("access").alias("use"), + PossibleValue::new("ctime").alias("status"), + PossibleValue::new("creation").alias("birth"), + ])) .help( "show time of the last modification of any file in the \ directory, or any of its subdirectories. If WORD is given, show time as WORD instead \ @@ -1100,10 +1094,12 @@ fn format_error_message(error: &ParseSizeError, s: &str, option: &str) -> String // GNU's du echos affected flag, -B or --block-size (-t or --threshold), depending user's selection match error { ParseSizeError::InvalidSuffix(_) => { - format!("invalid suffix in --{} argument {}", option, s.quote()) + format!("invalid suffix in --{option} argument {}", s.quote()) + } + ParseSizeError::ParseFailure(_) | ParseSizeError::PhysicalMem(_) => { + format!("invalid --{option} argument {}", s.quote()) } - ParseSizeError::ParseFailure(_) => format!("invalid --{} argument {}", option, s.quote()), - ParseSizeError::SizeTooBig(_) => format!("--{} argument {} too large", option, s.quote()), + ParseSizeError::SizeTooBig(_) => format!("--{option} argument {} too large", s.quote()), } } diff --git a/src/uu/echo/Cargo.toml b/src/uu/echo/Cargo.toml index ddb896c3389..80d56368749 100644 --- a/src/uu/echo/Cargo.toml +++ b/src/uu/echo/Cargo.toml @@ -1,22 +1,25 @@ [package] name = "uu_echo" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "echo ~ (uutils) display TEXT" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/echo" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/echo.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["format"] } [[bin]] name = "echo" diff --git a/src/uu/echo/src/echo.rs b/src/uu/echo/src/echo.rs index a34c99bc91d..4df76634843 100644 --- a/src/uu/echo/src/echo.rs +++ b/src/uu/echo/src/echo.rs @@ -3,12 +3,13 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -use clap::{crate_version, Arg, ArgAction, Command}; -use std::io::{self, Write}; -use std::iter::Peekable; -use std::ops::ControlFlow; -use std::str::Chars; -use uucore::error::{FromIo, UResult}; +use clap::builder::ValueParser; +use clap::{Arg, ArgAction, Command}; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::io::{self, StdoutLock, Write}; +use uucore::error::{UResult, USimpleError}; +use uucore::format::{FormatChar, OctalParsing, parse_escape_only}; use uucore::{format_usage, help_about, help_section, help_usage}; const ABOUT: &str = help_about!("echo.md"); @@ -22,124 +23,124 @@ mod options { pub const DISABLE_BACKSLASH_ESCAPE: &str = "disable_backslash_escape"; } -#[repr(u8)] -#[derive(Clone, Copy)] -enum Base { - Oct = 8, - Hex = 16, +/// Holds the options for echo command: +/// -n (disable newline) +/// -e/-E (escape handling), +struct EchoOptions { + /// -n flag option: if true, output a trailing newline (-n disables it) + /// Default: true + pub trailing_newline: bool, + + /// -e enables escape interpretation, -E disables it + /// Default: false (escape interpretation disabled) + pub escape: bool, } -impl Base { - fn max_digits(&self) -> u8 { - match self { - Self::Oct => 3, - Self::Hex => 2, +/// Checks if an argument is a valid echo flag +/// Returns true if valid echo flag found +fn is_echo_flag(arg: &OsString, echo_options: &mut EchoOptions) -> bool { + let bytes = arg.as_encoded_bytes(); + if bytes.first() == Some(&b'-') && arg != "-" { + // we initialize our local variables to the "current" options so we don't override + // previous found flags + let mut escape = echo_options.escape; + let mut trailing_newline = echo_options.trailing_newline; + + // Process characters after the '-' + for c in &bytes[1..] { + match c { + b'e' => escape = true, + b'E' => escape = false, + b'n' => trailing_newline = false, + // if there is any char in an argument starting with '-' that doesn't match e/E/n + // present means that this argument is not a flag + _ => return false, + } } - } -} -/// Parse the numeric part of the `\xHHH` and `\0NNN` escape sequences -fn parse_code(input: &mut Peekable, base: Base) -> Option { - // All arithmetic on `ret` needs to be wrapping, because octal input can - // take 3 digits, which is 9 bits, and therefore more than what fits in a - // `u8`. GNU just seems to wrap these values. - // Note that if we instead make `ret` a `u32` and use `char::from_u32` will - // yield incorrect results because it will interpret values larger than - // `u8::MAX` as unicode. - let mut ret = input.peek().and_then(|c| c.to_digit(base as u32))? as u8; - - // We can safely ignore the None case because we just peeked it. - let _ = input.next(); - - for _ in 1..base.max_digits() { - match input.peek().and_then(|c| c.to_digit(base as u32)) { - Some(n) => ret = ret.wrapping_mul(base as u8).wrapping_add(n as u8), - None => break, - } - // We can safely ignore the None case because we just peeked it. - let _ = input.next(); + // we only override the options with flags being found once we parsed the whole argument + echo_options.escape = escape; + echo_options.trailing_newline = trailing_newline; + return true; } - Some(ret.into()) + // argument doesn't start with '-' or is "-" => no flag + false } -fn print_escaped(input: &str, mut output: impl Write) -> io::Result> { - let mut iter = input.chars().peekable(); - while let Some(c) = iter.next() { - if c != '\\' { - write!(output, "{c}")?; - continue; - } - - // This is for the \NNN syntax for octal sequences. - // Note that '0' is intentionally omitted because that - // would be the \0NNN syntax. - if let Some('1'..='8') = iter.peek() { - if let Some(parsed) = parse_code(&mut iter, Base::Oct) { - write!(output, "{parsed}")?; - continue; - } - } - - if let Some(next) = iter.next() { - let unescaped = match next { - '\\' => '\\', - 'a' => '\x07', - 'b' => '\x08', - 'c' => return Ok(ControlFlow::Break(())), - 'e' => '\x1b', - 'f' => '\x0c', - 'n' => '\n', - 'r' => '\r', - 't' => '\t', - 'v' => '\x0b', - 'x' => { - if let Some(c) = parse_code(&mut iter, Base::Hex) { - c - } else { - write!(output, "\\")?; - 'x' - } - } - '0' => parse_code(&mut iter, Base::Oct).unwrap_or('\0'), - c => { - write!(output, "\\")?; - c - } - }; - write!(output, "{unescaped}")?; - } else { - write!(output, "\\")?; +/// Processes command line arguments, separating flags from normal arguments +/// Returns: +/// - Vector of non-flag arguments +/// - trailing_newline: whether to print a trailing newline +/// - escape: whether to process escape sequences +fn filter_echo_flags(args: impl uucore::Args) -> (Vec, bool, bool) { + let mut result = Vec::new(); + let mut echo_options = EchoOptions { + trailing_newline: true, + escape: false, + }; + let mut args_iter = args.into_iter(); + + // Process arguments until first non-flag is found + for arg in &mut args_iter { + // we parse flags and store options found in "echo_option". First is_echo_flag + // call to return false will break the loop and we will collect the remaining arguments + if !is_echo_flag(&arg, &mut echo_options) { + // First non-flag argument stops flag processing + result.push(arg); + break; } } - - Ok(ControlFlow::Continue(())) + // Collect remaining arguments + for arg in args_iter { + result.push(arg); + } + (result, echo_options.trailing_newline, echo_options.escape) } #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let matches = uu_app().get_matches_from(args); - - let no_newline = matches.get_flag(options::NO_NEWLINE); - let escaped = matches.get_flag(options::ENABLE_BACKSLASH_ESCAPE); - let values: Vec = match matches.get_many::(options::STRING) { - Some(s) => s.map(|s| s.to_string()).collect(), - None => vec![String::new()], + // Check POSIX compatibility mode + let is_posixly_correct = env::var_os("POSIXLY_CORRECT").is_some(); + + let args_iter = args.skip(1); + let (args, trailing_newline, escaped) = if is_posixly_correct { + let mut args_iter = args_iter.peekable(); + + if args_iter.peek() == Some(&OsString::from("-n")) { + // if POSIXLY_CORRECT is set and the first argument is the "-n" flag + // we filter flags normally but 'escaped' is activated nonetheless + let (args, _, _) = filter_echo_flags(args_iter); + (args, false, true) + } else { + // if POSIXLY_CORRECT is set and the first argument is not the "-n" flag + // we just collect all arguments as every argument is considered an argument + let args: Vec = args_iter.collect(); + (args, true, true) + } + } else { + // if POSIXLY_CORRECT is not set we filter the flags normally + let (args, trailing_newline, escaped) = filter_echo_flags(args_iter); + (args, trailing_newline, escaped) }; - execute(no_newline, escaped, &values) - .map_err_context(|| "could not write to stdout".to_string()) + let mut stdout_lock = io::stdout().lock(); + execute(&mut stdout_lock, args, trailing_newline, escaped)?; + + Ok(()) } pub fn uu_app() -> Command { + // Note: echo is different from the other utils in that it should **not** + // have `infer_long_args(true)`, because, for example, `--ver` should be + // printed as `--ver` and not show the version text. Command::new(uucore::util_name()) // TrailingVarArg specifies the final positional argument is a VarArg // and it doesn't attempts the parse any further args. // Final argument must have multiple(true) or the usage string equivalent. .trailing_var_arg(true) .allow_hyphen_values(true) - .infer_long_args(true) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .after_help(AFTER_HELP) .override_usage(format_usage(USAGE)) @@ -163,29 +164,61 @@ pub fn uu_app() -> Command { .action(ArgAction::SetTrue) .overrides_with(options::ENABLE_BACKSLASH_ESCAPE), ) - .arg(Arg::new(options::STRING).action(ArgAction::Append)) + .arg( + Arg::new(options::STRING) + .action(ArgAction::Append) + .value_parser(ValueParser::os_string()), + ) } -fn execute(no_newline: bool, escaped: bool, free: &[String]) -> io::Result<()> { - let stdout = io::stdout(); - let mut output = stdout.lock(); +fn execute( + stdout_lock: &mut StdoutLock, + arguments_after_options: Vec, + trailing_newline: bool, + escaped: bool, +) -> UResult<()> { + for (i, input) in arguments_after_options.into_iter().enumerate() { + let Some(bytes) = bytes_from_os_string(input.as_os_str()) else { + return Err(USimpleError::new( + 1, + "Non-UTF-8 arguments provided, but this platform does not support them", + )); + }; - for (i, input) in free.iter().enumerate() { if i > 0 { - write!(output, " ")?; + stdout_lock.write_all(b" ")?; } + if escaped { - if print_escaped(input, &mut output)?.is_break() { - return Ok(()); + for item in parse_escape_only(bytes, OctalParsing::ThreeDigits) { + if item.write(&mut *stdout_lock)?.is_break() { + return Ok(()); + } } } else { - write!(output, "{input}")?; + stdout_lock.write_all(bytes)?; } } - if !no_newline { - writeln!(output)?; + if trailing_newline { + stdout_lock.write_all(b"\n")?; } Ok(()) } + +fn bytes_from_os_string(input: &OsStr) -> Option<&[u8]> { + #[cfg(target_family = "unix")] + { + use std::os::unix::ffi::OsStrExt; + + Some(input.as_bytes()) + } + + #[cfg(not(target_family = "unix"))] + { + // TODO + // Verify that this works correctly on these platforms + input.to_str().map(|st| st.as_bytes()) + } +} diff --git a/src/uu/env/Cargo.toml b/src/uu/env/Cargo.toml index 26fa1ebbec8..c943ef4851a 100644 --- a/src/uu/env/Cargo.toml +++ b/src/uu/env/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_env" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "env ~ (uutils) set each NAME to VALUE in the environment and run COMMAND" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/env" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/env.rs" @@ -17,6 +20,7 @@ path = "src/env.rs" [dependencies] clap = { workspace = true } rust-ini = { workspace = true } +thiserror = { workspace = true } uucore = { workspace = true, features = ["signals"] } [target.'cfg(unix)'.dependencies] diff --git a/src/uu/env/src/env.rs b/src/uu/env/src/env.rs index 608357f5050..51479b5902f 100644 --- a/src/uu/env/src/env.rs +++ b/src/uu/env/src/env.rs @@ -5,34 +5,104 @@ // spell-checker:ignore (ToDO) chdir execvp progname subcommand subcommands unsets setenv putenv spawnp SIGSEGV SIGBUS sigaction -use clap::{crate_name, crate_version, Arg, ArgAction, Command}; +pub mod native_int_str; +pub mod split_iterator; +pub mod string_expander; +pub mod string_parser; +pub mod variable_parser; + +use clap::builder::ValueParser; +use clap::{Arg, ArgAction, Command, crate_name}; use ini::Ini; +use native_int_str::{ + Convert, NCvt, NativeIntStr, NativeIntString, NativeStr, from_native_int_representation_owned, +}; #[cfg(unix)] -use nix::sys::signal::{raise, sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal}; +use nix::sys::signal::{ + SaFlags, SigAction, SigHandler, SigHandler::SigIgn, SigSet, Signal, raise, sigaction, signal, +}; use std::borrow::Cow; use std::env; +use std::ffi::{OsStr, OsString}; use std::io::{self, Write}; -use std::iter::Iterator; + +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; #[cfg(unix)] -use std::os::unix::process::ExitStatusExt; -use std::process; +use std::os::unix::process::{CommandExt, ExitStatusExt}; +use std::process::{self}; use uucore::display::Quotable; -use uucore::error::{UClapError, UResult, USimpleError, UUsageError}; +use uucore::error::{ExitCode, UError, UResult, USimpleError, UUsageError}; use uucore::line_ending::LineEnding; +#[cfg(unix)] +use uucore::signals::signal_by_name_or_value; use uucore::{format_usage, help_about, help_section, help_usage, show_warning}; +use thiserror::Error; + +#[derive(Debug, Error, PartialEq)] +pub enum EnvError { + #[error("no terminating quote in -S string")] + EnvMissingClosingQuote(usize, char), + #[error("invalid backslash at end of string in -S")] + EnvInvalidBackslashAtEndOfStringInMinusS(usize, String), + #[error("'\\c' must not appear in double-quoted -S string")] + EnvBackslashCNotAllowedInDoubleQuotes(usize), + #[error("invalid sequence '\\{}' in -S",.1)] + EnvInvalidSequenceBackslashXInMinusS(usize, char), + #[error("Missing closing brace")] + EnvParsingOfVariableMissingClosingBrace(usize), + #[error("Missing variable name")] + EnvParsingOfMissingVariable(usize), + #[error("Missing closing brace after default value at {}",.0)] + EnvParsingOfVariableMissingClosingBraceAfterValue(usize), + #[error("Unexpected character: '{}', expected variable name must not start with 0..9",.1)] + EnvParsingOfVariableUnexpectedNumber(usize, String), + #[error("Unexpected character: '{}', expected a closing brace ('}}') or colon (':')",.1)] + EnvParsingOfVariableExceptedBraceOrColon(usize, String), + #[error("")] + EnvReachedEnd, + #[error("")] + EnvContinueWithDelimiter, + #[error("{}{:?}",.0,.1)] + EnvInternalError(usize, string_parser::Error), +} + +impl From for EnvError { + fn from(value: string_parser::Error) -> Self { + EnvError::EnvInternalError(value.peek_position, value) + } +} + const ABOUT: &str = help_about!("env.md"); const USAGE: &str = help_usage!("env.md"); const AFTER_HELP: &str = help_section!("after help", "env.md"); +mod options { + pub const IGNORE_ENVIRONMENT: &str = "ignore-environment"; + pub const CHDIR: &str = "chdir"; + pub const NULL: &str = "null"; + pub const FILE: &str = "file"; + pub const UNSET: &str = "unset"; + pub const DEBUG: &str = "debug"; + pub const SPLIT_STRING: &str = "split-string"; + pub const ARGV0: &str = "argv0"; + pub const IGNORE_SIGNAL: &str = "ignore-signal"; +} + +const ERROR_MSG_S_SHEBANG: &str = "use -[v]S to pass options in shebang lines"; + struct Options<'a> { ignore_env: bool, line_ending: LineEnding, - running_directory: Option<&'a str>, - files: Vec<&'a str>, - unsets: Vec<&'a str>, - sets: Vec<(&'a str, &'a str)>, - program: Vec<&'a str>, + running_directory: Option<&'a OsStr>, + files: Vec<&'a OsStr>, + unsets: Vec<&'a OsStr>, + sets: Vec<(Cow<'a, OsStr>, Cow<'a, OsStr>)>, + program: Vec<&'a OsStr>, + argv0: Option<&'a OsStr>, + #[cfg(unix)] + ignore_signal: Vec, } // print name=value env pairs on screen @@ -41,17 +111,17 @@ fn print_env(line_ending: LineEnding) { let stdout_raw = io::stdout(); let mut stdout = stdout_raw.lock(); for (n, v) in env::vars() { - write!(stdout, "{}={}{}", n, v, line_ending).unwrap(); + write!(stdout, "{n}={v}{line_ending}").unwrap(); } } -fn parse_name_value_opt<'a>(opts: &mut Options<'a>, opt: &'a str) -> UResult { +fn parse_name_value_opt<'a>(opts: &mut Options<'a>, opt: &'a OsStr) -> UResult { // is it a NAME=VALUE like opt ? - if let Some(idx) = opt.find('=') { + let wrap = NativeStr::<'a>::new(opt); + let split_o = wrap.split_once(&'='); + if let Some((name, value)) = split_o { // yes, so push name, value pair - let (name, value) = opt.split_at(idx); - opts.sets.push((name, &value['='.len_utf8()..])); - + opts.sets.push((name, value)); Ok(false) } else { // no, it's a program-like opt @@ -59,7 +129,7 @@ fn parse_name_value_opt<'a>(opts: &mut Options<'a>, opt: &'a str) -> UResult(opts: &mut Options<'a>, opt: &'a str) -> UResult<()> { +fn parse_program_opt<'a>(opts: &mut Options<'a>, opt: &'a OsStr) -> UResult<()> { if opts.line_ending == LineEnding::Nul { Err(UUsageError::new( 125, @@ -71,6 +141,56 @@ fn parse_program_opt<'a>(opts: &mut Options<'a>, opt: &'a str) -> UResult<()> { } } +#[cfg(unix)] +fn parse_signal_value(signal_name: &str) -> UResult { + let signal_name_upcase = signal_name.to_uppercase(); + let optional_signal_value = signal_by_name_or_value(&signal_name_upcase); + let error = USimpleError::new(125, format!("{}: invalid signal", signal_name.quote())); + match optional_signal_value { + Some(sig_val) => { + if sig_val == 0 { + Err(error) + } else { + Ok(sig_val) + } + } + None => Err(error), + } +} + +#[cfg(unix)] +fn parse_signal_opt<'a>(opts: &mut Options<'a>, opt: &'a OsStr) -> UResult<()> { + if opt.is_empty() { + return Ok(()); + } + let signals: Vec<&'a OsStr> = opt + .as_bytes() + .split(|&b| b == b',') + .map(OsStr::from_bytes) + .collect(); + + let mut sig_vec = Vec::with_capacity(signals.len()); + for sig in signals { + if !sig.is_empty() { + sig_vec.push(sig); + } + } + for sig in sig_vec { + let Some(sig_str) = sig.to_str() else { + return Err(USimpleError::new( + 1, + format!("{}: invalid signal", sig.quote()), + )); + }; + let sig_val = parse_signal_value(sig_str)?; + if !opts.ignore_signal.contains(&sig_val) { + opts.ignore_signal.push(sig_val); + } + } + + Ok(()) +} + fn load_config_file(opts: &mut Options) -> UResult<()> { // NOTE: config files are parsed using an INI parser b/c it's available and compatible with ".env"-style files // ... * but support for actual INI files, although working, is not intended, nor claimed @@ -84,12 +204,14 @@ fn load_config_file(opts: &mut Options) -> UResult<()> { }; let conf = - conf.map_err(|e| USimpleError::new(1, format!("{}: {}", file.maybe_quote(), e)))?; + conf.map_err(|e| USimpleError::new(1, format!("{}: {e}", file.maybe_quote())))?; for (_, prop) in &conf { // ignore all INI section lines (treat them as comments) - for (key, value) in prop.iter() { - env::set_var(key, value); + for (key, value) in prop { + unsafe { + env::set_var(key, value); + } } } } @@ -97,51 +219,35 @@ fn load_config_file(opts: &mut Options) -> UResult<()> { Ok(()) } -#[cfg(not(windows))] -#[allow(clippy::ptr_arg)] -fn build_command<'a, 'b>(args: &'a Vec<&'b str>) -> (Cow<'b, str>, &'a [&'b str]) { - let progname = Cow::from(args[0]); - (progname, &args[1..]) -} - -#[cfg(windows)] -fn build_command<'a, 'b>(args: &'a mut Vec<&'b str>) -> (Cow<'b, str>, &'a [&'b str]) { - args.insert(0, "/d/c"); - let progname = env::var("ComSpec") - .map(Cow::from) - .unwrap_or_else(|_| Cow::from("cmd")); - - (progname, &args[..]) -} - pub fn uu_app() -> Command { Command::new(crate_name!()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .after_help(AFTER_HELP) .infer_long_args(true) .trailing_var_arg(true) .arg( - Arg::new("ignore-environment") + Arg::new(options::IGNORE_ENVIRONMENT) .short('i') - .long("ignore-environment") + .long(options::IGNORE_ENVIRONMENT) .help("start with an empty environment") .action(ArgAction::SetTrue), ) .arg( - Arg::new("chdir") + Arg::new(options::CHDIR) .short('C') // GNU env compatibility - .long("chdir") + .long(options::CHDIR) .number_of_values(1) .value_name("DIR") + .value_parser(ValueParser::os_string()) .value_hint(clap::ValueHint::DirPath) .help("change working directory to DIR"), ) .arg( - Arg::new("null") + Arg::new(options::NULL) .short('0') - .long("null") + .long(options::NULL) .help( "end each output line with a 0 byte rather than a newline (only \ valid when printing the environment)", @@ -149,11 +255,12 @@ pub fn uu_app() -> Command { .action(ArgAction::SetTrue), ) .arg( - Arg::new("file") + Arg::new(options::FILE) .short('f') - .long("file") + .long(options::FILE) .value_name("PATH") .value_hint(clap::ValueHint::FilePath) + .value_parser(ValueParser::os_string()) .action(ArgAction::Append) .help( "read and set variables from a \".env\"-style configuration file \ @@ -161,32 +268,406 @@ pub fn uu_app() -> Command { ), ) .arg( - Arg::new("unset") + Arg::new(options::UNSET) .short('u') - .long("unset") + .long(options::UNSET) .value_name("NAME") .action(ArgAction::Append) + .value_parser(ValueParser::os_string()) .help("remove variable from the environment"), ) - .arg(Arg::new("vars").action(ArgAction::Append)) + .arg( + Arg::new(options::DEBUG) + .short('v') + .long(options::DEBUG) + .action(ArgAction::Count) + .help("print verbose information for each processing step"), + ) + .arg( + Arg::new(options::SPLIT_STRING) // split string handling is implemented directly, not using CLAP. But this entry here is needed for the help information output. + .short('S') + .long(options::SPLIT_STRING) + .value_name("S") + .action(ArgAction::Set) + .value_parser(ValueParser::os_string()) + .help("process and split S into separate arguments; used to pass multiple arguments on shebang lines") + ).arg( + Arg::new(options::ARGV0) + .overrides_with(options::ARGV0) + .short('a') + .long(options::ARGV0) + .value_name("a") + .action(ArgAction::Set) + .value_parser(ValueParser::os_string()) + .help("Override the zeroth argument passed to the command being executed. \ + Without this option a default value of `command` is used.") + ) + .arg( + Arg::new("vars") + .action(ArgAction::Append) + .value_parser(ValueParser::os_string()) + ) + .arg( + Arg::new(options::IGNORE_SIGNAL) + .long(options::IGNORE_SIGNAL) + .value_name("SIG") + .action(ArgAction::Append) + .value_parser(ValueParser::os_string()) + .help("set handling of SIG signal(s) to do nothing") + ) +} + +pub fn parse_args_from_str(text: &NativeIntStr) -> UResult> { + split_iterator::split(text).map_err(|e| match e { + EnvError::EnvBackslashCNotAllowedInDoubleQuotes(_) => USimpleError::new(125, e.to_string()), + EnvError::EnvInvalidBackslashAtEndOfStringInMinusS(_, _) => { + USimpleError::new(125, e.to_string()) + } + EnvError::EnvInvalidSequenceBackslashXInMinusS(_, _) => { + USimpleError::new(125, e.to_string()) + } + EnvError::EnvMissingClosingQuote(_, _) => USimpleError::new(125, e.to_string()), + EnvError::EnvParsingOfVariableMissingClosingBrace(pos) => { + USimpleError::new(125, format!("variable name issue (at {pos}): {e}")) + } + EnvError::EnvParsingOfMissingVariable(pos) => { + USimpleError::new(125, format!("variable name issue (at {pos}): {e}")) + } + EnvError::EnvParsingOfVariableMissingClosingBraceAfterValue(pos) => { + USimpleError::new(125, format!("variable name issue (at {pos}): {e}")) + } + EnvError::EnvParsingOfVariableUnexpectedNumber(pos, _) => { + USimpleError::new(125, format!("variable name issue (at {pos}): {e}")) + } + EnvError::EnvParsingOfVariableExceptedBraceOrColon(pos, _) => { + USimpleError::new(125, format!("variable name issue (at {pos}): {e}")) + } + _ => USimpleError::new(125, format!("Error: {e:?}")), + }) +} + +fn debug_print_args(args: &[OsString]) { + eprintln!("input args:"); + for (i, arg) in args.iter().enumerate() { + eprintln!("arg[{i}]: {}", arg.quote()); + } +} + +fn check_and_handle_string_args( + arg: &OsString, + prefix_to_test: &str, + all_args: &mut Vec, + do_debug_print_args: Option<&Vec>, +) -> UResult { + let native_arg = NCvt::convert(arg); + if let Some(remaining_arg) = native_arg.strip_prefix(&*NCvt::convert(prefix_to_test)) { + if let Some(input_args) = do_debug_print_args { + debug_print_args(input_args); // do it here, such that its also printed when we get an error/panic during parsing + } + + let arg_strings = parse_args_from_str(remaining_arg)?; + all_args.extend( + arg_strings + .into_iter() + .map(from_native_int_representation_owned), + ); + + Ok(true) + } else { + Ok(false) + } +} + +#[derive(Default)] +struct EnvAppData { + do_debug_printing: bool, + do_input_debug_printing: Option, + had_string_argument: bool, } -#[allow(clippy::cognitive_complexity)] -fn run_env(args: impl uucore::Args) -> UResult<()> { - let app = uu_app(); - let matches = app.try_get_matches_from(args).with_exit_code(125)?; +impl EnvAppData { + fn make_error_no_such_file_or_dir(&self, prog: &OsStr) -> Box { + uucore::show_error!("{}: No such file or directory", prog.quote()); + if !self.had_string_argument { + uucore::show_error!("{ERROR_MSG_S_SHEBANG}"); + } + ExitCode::new(127) + } + + fn process_all_string_arguments( + &mut self, + original_args: &Vec, + ) -> UResult> { + let mut all_args: Vec = Vec::new(); + let mut process_flags = true; + let mut expecting_arg = false; + // Leave out split-string since it's a special case below + let flags_with_args = [ + options::ARGV0, + options::CHDIR, + options::FILE, + options::IGNORE_SIGNAL, + options::UNSET, + ]; + let short_flags_with_args = ['a', 'C', 'f', 'u']; + for (n, arg) in original_args.iter().enumerate() { + let arg_str = arg.to_string_lossy(); + // Stop processing env flags once we reach the command or -- argument + if 0 < n + && !expecting_arg + && (arg == "--" || !(arg_str.starts_with('-') || arg_str.contains('='))) + { + process_flags = false; + } + if !process_flags { + all_args.push(arg.clone()); + continue; + } + expecting_arg = false; + match arg { + b if check_and_handle_string_args(b, "--split-string", &mut all_args, None)? => { + self.had_string_argument = true; + } + b if check_and_handle_string_args(b, "-S", &mut all_args, None)? => { + self.had_string_argument = true; + } + b if check_and_handle_string_args(b, "-vS", &mut all_args, None)? => { + self.do_debug_printing = true; + self.had_string_argument = true; + } + b if check_and_handle_string_args( + b, + "-vvS", + &mut all_args, + Some(original_args), + )? => + { + self.do_debug_printing = true; + self.do_input_debug_printing = Some(false); // already done + self.had_string_argument = true; + } + _ => { + if let Some(flag) = arg_str.strip_prefix("--") { + if flags_with_args.contains(&flag) { + expecting_arg = true; + } + } else if let Some(flag) = arg_str.strip_prefix("-") { + for c in flag.chars() { + expecting_arg = short_flags_with_args.contains(&c); + } + } + // Short unset option (-u) is not allowed to contain '=' + if arg_str.contains('=') + && arg_str.starts_with("-u") + && !arg_str.starts_with("--") + { + return Err(USimpleError::new( + 125, + format!("cannot unset '{}': Invalid argument", &arg_str[2..]), + )); + } + + all_args.push(arg.clone()); + } + } + } + + Ok(all_args) + } + + fn parse_arguments( + &mut self, + original_args: impl uucore::Args, + ) -> Result<(Vec, clap::ArgMatches), Box> { + let original_args: Vec = original_args.collect(); + let args = self.process_all_string_arguments(&original_args)?; + let app = uu_app(); + let matches = app + .try_get_matches_from(args) + .map_err(|e| -> Box { + match e.kind() { + clap::error::ErrorKind::DisplayHelp + | clap::error::ErrorKind::DisplayVersion => e.into(), + _ => { + // extent any real issue with parameter parsing by the ERROR_MSG_S_SHEBANG + let s = format!("{e}"); + if !s.is_empty() { + let s = s.trim_end(); + uucore::show_error!("{s}"); + } + uucore::show_error!("{ERROR_MSG_S_SHEBANG}"); + ExitCode::new(125) + } + } + })?; + Ok((original_args, matches)) + } + + fn run_env(&mut self, original_args: impl uucore::Args) -> UResult<()> { + let (original_args, matches) = self.parse_arguments(original_args)?; + + self.do_debug_printing = self.do_debug_printing || (0 != matches.get_count("debug")); + self.do_input_debug_printing = self + .do_input_debug_printing + .or(Some(matches.get_count("debug") >= 2)); + if let Some(value) = self.do_input_debug_printing { + if value { + debug_print_args(&original_args); + self.do_input_debug_printing = Some(false); + } + } + + let mut opts = make_options(&matches)?; + + apply_change_directory(&opts)?; + + // NOTE: we manually set and unset the env vars below rather than using Command::env() to more + // easily handle the case where no command is given + + apply_removal_of_all_env_vars(&opts); + + // load .env-style config file prior to those given on the command-line + load_config_file(&mut opts)?; + + apply_unset_env_vars(&opts)?; + + apply_specified_env_vars(&opts); + + #[cfg(unix)] + apply_ignore_signal(&opts)?; + + if opts.program.is_empty() { + // no program provided, so just dump all env vars to stdout + print_env(opts.line_ending); + } else { + return self.run_program(&opts, self.do_debug_printing); + } + Ok(()) + } + + fn run_program( + &mut self, + opts: &Options<'_>, + do_debug_printing: bool, + ) -> Result<(), Box> { + let prog = Cow::from(opts.program[0]); + #[cfg(unix)] + let mut arg0 = prog.clone(); + #[cfg(not(unix))] + let arg0 = prog.clone(); + let args = &opts.program[1..]; + + /* + * On Unix-like systems Command::status either ends up calling either fork or posix_spawnp + * (which ends up calling clone). Keep using the current process would be ideal, but the + * standard library contains many checks and fail-safes to ensure the process ends up being + * created. This is much simpler than dealing with the hassles of calling execvp directly. + */ + let mut cmd = process::Command::new(&*prog); + cmd.args(args); + + if let Some(_argv0) = opts.argv0 { + #[cfg(unix)] + { + cmd.arg0(_argv0); + arg0 = Cow::Borrowed(_argv0); + if do_debug_printing { + eprintln!("argv0: {}", arg0.quote()); + } + } + + #[cfg(not(unix))] + return Err(USimpleError::new( + 2, + "--argv0 is currently not supported on this platform", + )); + } + + if do_debug_printing { + eprintln!("executing: {}", prog.maybe_quote()); + let arg_prefix = " arg"; + eprintln!("{arg_prefix}[{}]= {}", 0, arg0.quote()); + for (i, arg) in args.iter().enumerate() { + eprintln!("{arg_prefix}[{}]= {}", i + 1, arg.quote()); + } + } + + match cmd.status() { + Ok(exit) if !exit.success() => { + #[cfg(unix)] + if let Some(exit_code) = exit.code() { + return Err(exit_code.into()); + } else { + // `exit.code()` returns `None` on Unix when the process is terminated by a signal. + // See std::os::unix::process::ExitStatusExt for more information. This prints out + // the interrupted process and the signal it received. + let signal_code = exit.signal().unwrap(); + let signal = Signal::try_from(signal_code).unwrap(); + + // We have to disable any handler that's installed by default. + // This ensures that we exit on this signal. + // For example, `SIGSEGV` and `SIGBUS` have default handlers installed in Rust. + // We ignore the errors because there is not much we can do if that fails anyway. + // SAFETY: The function is unsafe because installing functions is unsafe, but we are + // just defaulting to default behavior and not installing a function. Hence, the call + // is safe. + let _ = unsafe { + sigaction( + signal, + &SigAction::new(SigHandler::SigDfl, SaFlags::empty(), SigSet::all()), + ) + }; + + let _ = raise(signal); + } + return Err(exit.code().unwrap().into()); + } + Err(ref err) => { + return match err.kind() { + io::ErrorKind::NotFound | io::ErrorKind::InvalidInput => { + Err(self.make_error_no_such_file_or_dir(&prog)) + } + io::ErrorKind::PermissionDenied => { + uucore::show_error!("{}: Permission denied", prog.quote()); + Err(126.into()) + } + _ => { + uucore::show_error!("unknown error: {err:?}"); + Err(126.into()) + } + }; + } + Ok(_) => (), + } + Ok(()) + } +} + +fn apply_removal_of_all_env_vars(opts: &Options<'_>) { + // remove all env vars if told to ignore presets + if opts.ignore_env { + for (ref name, _) in env::vars_os() { + unsafe { + env::remove_var(name); + } + } + } +} + +fn make_options(matches: &clap::ArgMatches) -> UResult> { let ignore_env = matches.get_flag("ignore-environment"); let line_ending = LineEnding::from_zero_flag(matches.get_flag("null")); - let running_directory = matches.get_one::("chdir").map(|s| s.as_str()); - let files = match matches.get_many::("file") { - Some(v) => v.map(|s| s.as_str()).collect(), + let running_directory = matches.get_one::("chdir").map(|s| s.as_os_str()); + let files = match matches.get_many::("file") { + Some(v) => v.map(|s| s.as_os_str()).collect(), None => Vec::with_capacity(0), }; - let unsets = match matches.get_many::("unset") { - Some(v) => v.map(|s| s.as_str()).collect(), + let unsets = match matches.get_many::("unset") { + Some(v) => v.map(|s| s.as_os_str()).collect(), None => Vec::with_capacity(0), }; + let argv0 = matches.get_one::("argv0").map(|s| s.as_os_str()); let mut opts = Options { ignore_env, @@ -196,23 +677,20 @@ fn run_env(args: impl uucore::Args) -> UResult<()> { unsets, sets: vec![], program: vec![], + argv0, + #[cfg(unix)] + ignore_signal: vec![], }; - // change directory - if let Some(d) = opts.running_directory { - match env::set_current_dir(d) { - Ok(()) => d, - Err(error) => { - return Err(USimpleError::new( - 125, - format!("cannot change directory to \"{d}\": {error}"), - )); - } - }; + #[cfg(unix)] + if let Some(iter) = matches.get_many::("ignore-signal") { + for opt in iter { + parse_signal_opt(&mut opts, opt)?; + } } let mut begin_prog_opts = false; - if let Some(mut iter) = matches.get_many::("vars") { + if let Some(mut iter) = matches.get_many::("vars") { // read NAME=VALUE arguments (and up to a single program argument) while !begin_prog_opts { if let Some(opt) = iter.next() { @@ -232,41 +710,54 @@ fn run_env(args: impl uucore::Args) -> UResult<()> { } } - // GNU env tests this behavior - if opts.program.is_empty() && running_directory.is_some() { - return Err(UUsageError::new( - 125, - "must specify command with --chdir (-C)".to_string(), - )); - } - - // NOTE: we manually set and unset the env vars below rather than using Command::env() to more - // easily handle the case where no command is given - - // remove all env vars if told to ignore presets - if opts.ignore_env { - for (ref name, _) in env::vars() { - env::remove_var(name); - } - } - - // load .env-style config file prior to those given on the command-line - load_config_file(&mut opts)?; + Ok(opts) +} - // unset specified env vars +fn apply_unset_env_vars(opts: &Options<'_>) -> Result<(), Box> { for name in &opts.unsets { - if name.is_empty() || name.contains(0 as char) || name.contains('=') { + let native_name = NativeStr::new(name); + if name.is_empty() + || native_name.contains(&'\0').unwrap() + || native_name.contains(&'=').unwrap() + { return Err(USimpleError::new( 125, format!("cannot unset {}: Invalid argument", name.quote()), )); } + unsafe { + env::remove_var(name); + } + } + Ok(()) +} - env::remove_var(name); +fn apply_change_directory(opts: &Options<'_>) -> Result<(), Box> { + // GNU env tests this behavior + if opts.program.is_empty() && opts.running_directory.is_some() { + return Err(UUsageError::new( + 125, + "must specify command with --chdir (-C)".to_string(), + )); } + if let Some(d) = opts.running_directory { + match env::set_current_dir(d) { + Ok(()) => d, + Err(error) => { + return Err(USimpleError::new( + 125, + format!("cannot change directory to {}: {error}", d.quote()), + )); + } + }; + } + Ok(()) +} + +fn apply_specified_env_vars(opts: &Options<'_>) { // set specified env vars - for &(name, val) in &opts.sets { + for (name, val) in &opts.sets { /* * set_var panics if name is an empty string * set_var internally calls setenv (on unix at least), while GNU env calls putenv instead. @@ -293,66 +784,142 @@ fn run_env(args: impl uucore::Args) -> UResult<()> { show_warning!("no name specified for value {}", val.quote()); continue; } - env::set_var(name, val); + unsafe { + env::set_var(name, val); + } } +} - if opts.program.is_empty() { - // no program provided, so just dump all env vars to stdout - print_env(opts.line_ending); - } else { - // we need to execute a command - #[cfg(windows)] - let (prog, args) = build_command(&mut opts.program); - #[cfg(not(windows))] - let (prog, args) = build_command(&opts.program); - - /* - * On Unix-like systems Command::status either ends up calling either fork or posix_spawnp - * (which ends up calling clone). Keep using the current process would be ideal, but the - * standard library contains many checks and fail-safes to ensure the process ends up being - * created. This is much simpler than dealing with the hassles of calling execvp directly. - */ - match process::Command::new(&*prog).args(args).status() { - Ok(exit) if !exit.success() => { - #[cfg(unix)] - if let Some(exit_code) = exit.code() { - return Err(exit_code.into()); - } else { - // `exit.code()` returns `None` on Unix when the process is terminated by a signal. - // See std::os::unix::process::ExitStatusExt for more information. This prints out - // the interrupted process and the signal it received. - let signal_code = exit.signal().unwrap(); - let signal = Signal::try_from(signal_code).unwrap(); - - // We have to disable any handler that's installed by default. - // This ensures that we exit on this signal. - // For example, `SIGSEGV` and `SIGBUS` have default handlers installed in Rust. - // We ignore the errors because there is not much we can do if that fails anyway. - // SAFETY: The function is unsafe because installing functions is unsafe, but we are - // just defaulting to default behavior and not installing a function. Hence, the call - // is safe. - let _ = unsafe { - sigaction( - signal, - &SigAction::new(SigHandler::SigDfl, SaFlags::empty(), SigSet::all()), - ) - }; +#[cfg(unix)] +fn apply_ignore_signal(opts: &Options<'_>) -> UResult<()> { + for &sig_value in &opts.ignore_signal { + let sig: Signal = (sig_value as i32) + .try_into() + .map_err(|e| io::Error::from_raw_os_error(e as i32))?; - let _ = raise(signal); - } - #[cfg(not(unix))] - return Err(exit.code().unwrap().into()); - } - Err(ref err) if err.kind() == io::ErrorKind::NotFound => return Err(127.into()), - Err(_) => return Err(126.into()), - Ok(_) => (), - } + ignore_signal(sig)?; } + Ok(()) +} +#[cfg(unix)] +fn ignore_signal(sig: Signal) -> UResult<()> { + // SAFETY: This is safe because we write the handler for each signal only once, and therefore "the current handler is the default", as the documentation requires it. + let result = unsafe { signal(sig, SigIgn) }; + if let Err(err) = result { + return Err(USimpleError::new( + 125, + format!( + "failed to set signal action for signal {}: {}", + sig as i32, + err.desc() + ), + )); + } Ok(()) } #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - run_env(args) + EnvAppData::default().run_env(args) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_string_environment_vars_test() { + unsafe { env::set_var("FOO", "BAR") }; + assert_eq!( + NCvt::convert(vec!["FOO=bar", "sh", "-c", "echo xBARx =$FOO="]), + parse_args_from_str(&NCvt::convert(r#"FOO=bar sh -c "echo x${FOO}x =\$FOO=""#)) + .unwrap(), + ); + } + + #[test] + fn test_split_string_misc() { + assert_eq!( + NCvt::convert(vec!["A=B", "FOO=AR", "sh", "-c", "echo $A$FOO"]), + parse_args_from_str(&NCvt::convert(r#"A=B FOO=AR sh -c "echo \$A\$FOO""#)).unwrap(), + ); + assert_eq!( + NCvt::convert(vec!["A=B", "FOO=AR", "sh", "-c", "echo $A$FOO"]), + parse_args_from_str(&NCvt::convert(r"A=B FOO=AR sh -c 'echo $A$FOO'")).unwrap() + ); + assert_eq!( + NCvt::convert(vec!["A=B", "FOO=AR", "sh", "-c", "echo $A$FOO"]), + parse_args_from_str(&NCvt::convert(r"A=B FOO=AR sh -c 'echo $A$FOO'")).unwrap() + ); + + assert_eq!( + NCvt::convert(vec!["-i", "A=B ' C"]), + parse_args_from_str(&NCvt::convert(r"-i A='B \' C'")).unwrap() + ); + } + + #[test] + fn test_error_cases() { + // Test EnvBackslashCNotAllowedInDoubleQuotes + let result = parse_args_from_str(&NCvt::convert(r#"sh -c "echo \c""#)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "'\\c' must not appear in double-quoted -S string" + ); + + // Test EnvInvalidBackslashAtEndOfStringInMinusS + let result = parse_args_from_str(&NCvt::convert(r#"sh -c "echo \"#)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "no terminating quote in -S string" + ); + + // Test EnvInvalidSequenceBackslashXInMinusS + let result = parse_args_from_str(&NCvt::convert(r#"sh -c "echo \x""#)); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("invalid sequence '\\x' in -S") + ); + + // Test EnvMissingClosingQuote + let result = parse_args_from_str(&NCvt::convert(r#"sh -c "echo "#)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "no terminating quote in -S string" + ); + + // Test variable-related errors + let result = parse_args_from_str(&NCvt::convert(r"echo ${FOO")); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("variable name issue (at 10): Missing closing brace") + ); + + let result = parse_args_from_str(&NCvt::convert(r"echo ${FOO:-value")); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("variable name issue (at 17): Missing closing brace after default value") + ); + + let result = parse_args_from_str(&NCvt::convert(r"echo ${1FOO}")); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("variable name issue (at 7): Unexpected character: '1', expected variable name must not start with 0..9")); + + let result = parse_args_from_str(&NCvt::convert(r"echo ${FOO?}")); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("variable name issue (at 10): Unexpected character: '?', expected a closing brace ('}') or colon (':')")); + } } diff --git a/src/uu/env/src/native_int_str.rs b/src/uu/env/src/native_int_str.rs new file mode 100644 index 00000000000..856948fc137 --- /dev/null +++ b/src/uu/env/src/native_int_str.rs @@ -0,0 +1,316 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// This module contains classes and functions for dealing with the differences +// between operating systems regarding the lossless processing of OsStr/OsString. +// In contrast to existing crates with similar purpose, this module does not use any +// `unsafe` features or functions. +// Due to a suboptimal design aspect of OsStr/OsString on windows, we need to +// encode/decode to wide chars on windows operating system. +// This prevents borrowing from OsStr on windows. Anyway, if optimally used,# +// this conversion needs to be done only once in the beginning and at the end. + +use std::ffi::OsString; +#[cfg(not(target_os = "windows"))] +use std::os::unix::ffi::{OsStrExt, OsStringExt}; +#[cfg(target_os = "windows")] +use std::os::windows::prelude::*; +use std::{borrow::Cow, ffi::OsStr}; + +#[cfg(not(target_os = "windows"))] +use u8 as NativeIntCharU; +#[cfg(target_os = "windows")] +use u16 as NativeIntCharU; + +pub type NativeCharInt = NativeIntCharU; +pub type NativeIntStr = [NativeCharInt]; +pub type NativeIntString = Vec; + +pub struct NCvt; + +pub trait Convert { + fn convert(f: From) -> To; +} + +// ================ str/String ================= + +impl<'a> Convert<&'a str, Cow<'a, NativeIntStr>> for NCvt { + fn convert(f: &'a str) -> Cow<'a, NativeIntStr> { + #[cfg(target_os = "windows")] + { + Cow::Owned(f.encode_utf16().collect()) + } + + #[cfg(not(target_os = "windows"))] + { + Cow::Borrowed(f.as_bytes()) + } + } +} + +impl<'a> Convert<&'a String, Cow<'a, NativeIntStr>> for NCvt { + fn convert(f: &'a String) -> Cow<'a, NativeIntStr> { + #[cfg(target_os = "windows")] + { + Cow::Owned(f.encode_utf16().collect()) + } + + #[cfg(not(target_os = "windows"))] + { + Cow::Borrowed(f.as_bytes()) + } + } +} + +impl<'a> Convert> for NCvt { + fn convert(f: String) -> Cow<'a, NativeIntStr> { + #[cfg(target_os = "windows")] + { + Cow::Owned(f.encode_utf16().collect()) + } + + #[cfg(not(target_os = "windows"))] + { + Cow::Owned(f.into_bytes()) + } + } +} + +// ================ OsStr/OsString ================= + +impl<'a> Convert<&'a OsStr, Cow<'a, NativeIntStr>> for NCvt { + fn convert(f: &'a OsStr) -> Cow<'a, NativeIntStr> { + to_native_int_representation(f) + } +} + +impl<'a> Convert<&'a OsString, Cow<'a, NativeIntStr>> for NCvt { + fn convert(f: &'a OsString) -> Cow<'a, NativeIntStr> { + to_native_int_representation(f) + } +} + +impl<'a> Convert> for NCvt { + fn convert(f: OsString) -> Cow<'a, NativeIntStr> { + #[cfg(target_os = "windows")] + { + Cow::Owned(f.encode_wide().collect()) + } + + #[cfg(not(target_os = "windows"))] + { + Cow::Owned(f.into_vec()) + } + } +} + +// ================ Vec ================= + +impl<'a> Convert<&'a Vec<&'a str>, Vec>> for NCvt { + fn convert(f: &'a Vec<&'a str>) -> Vec> { + f.iter().map(|x| Self::convert(*x)).collect() + } +} + +impl<'a> Convert, Vec>> for NCvt { + fn convert(f: Vec<&'a str>) -> Vec> { + f.iter().map(|x| Self::convert(*x)).collect() + } +} + +impl<'a> Convert<&'a Vec, Vec>> for NCvt { + fn convert(f: &'a Vec) -> Vec> { + f.iter().map(Self::convert).collect() + } +} + +impl<'a> Convert, Vec>> for NCvt { + fn convert(f: Vec) -> Vec> { + f.into_iter().map(Self::convert).collect() + } +} + +pub fn to_native_int_representation(input: &OsStr) -> Cow<'_, NativeIntStr> { + #[cfg(target_os = "windows")] + { + Cow::Owned(input.encode_wide().collect()) + } + + #[cfg(not(target_os = "windows"))] + { + Cow::Borrowed(input.as_bytes()) + } +} + +#[allow(clippy::needless_pass_by_value)] // needed on windows +pub fn from_native_int_representation(input: Cow<'_, NativeIntStr>) -> Cow<'_, OsStr> { + #[cfg(target_os = "windows")] + { + Cow::Owned(OsString::from_wide(&input)) + } + + #[cfg(not(target_os = "windows"))] + { + match input { + Cow::Borrowed(borrow) => Cow::Borrowed(OsStr::from_bytes(borrow)), + Cow::Owned(own) => Cow::Owned(OsString::from_vec(own)), + } + } +} + +#[allow(clippy::needless_pass_by_value)] // needed on windows +pub fn from_native_int_representation_owned(input: NativeIntString) -> OsString { + #[cfg(target_os = "windows")] + { + OsString::from_wide(&input) + } + + #[cfg(not(target_os = "windows"))] + { + OsString::from_vec(input) + } +} + +pub fn get_single_native_int_value(c: &char) -> Option { + #[cfg(target_os = "windows")] + { + let mut buf = [0u16, 0]; + let s = c.encode_utf16(&mut buf); + if s.len() == 1 { Some(buf[0]) } else { None } + } + + #[cfg(not(target_os = "windows"))] + { + let mut buf = [0u8, 0, 0, 0]; + let s = c.encode_utf8(&mut buf); + if s.len() == 1 { Some(buf[0]) } else { None } + } +} + +pub fn get_char_from_native_int(ni: NativeCharInt) -> Option<(char, NativeCharInt)> { + let c_opt; + #[cfg(target_os = "windows")] + { + c_opt = char::decode_utf16([ni; 1]).next().unwrap().ok(); + }; + + #[cfg(not(target_os = "windows"))] + { + c_opt = std::str::from_utf8(&[ni; 1]) + .ok() + .map(|x| x.chars().next().unwrap()); + }; + + if let Some(c) = c_opt { + return Some((c, ni)); + } + + None +} + +pub struct NativeStr<'a> { + native: Cow<'a, NativeIntStr>, +} + +impl<'a> NativeStr<'a> { + pub fn new(str: &'a OsStr) -> Self { + Self { + native: to_native_int_representation(str), + } + } + + pub fn native(&self) -> Cow<'a, NativeIntStr> { + self.native.clone() + } + + pub fn into_native(self) -> Cow<'a, NativeIntStr> { + self.native + } + + pub fn contains(&self, x: &char) -> Option { + let n_c = get_single_native_int_value(x)?; + Some(self.native.contains(&n_c)) + } + + pub fn slice(&self, from: usize, to: usize) -> Cow<'a, OsStr> { + let result = self.match_cow(|b| Ok::<_, ()>(&b[from..to]), |o| Ok(o[from..to].to_vec())); + result.unwrap() + } + + pub fn split_once(&self, pred: &char) -> Option<(Cow<'a, OsStr>, Cow<'a, OsStr>)> { + let n_c = get_single_native_int_value(pred)?; + let p = self.native.iter().position(|&x| x == n_c)?; + let before = self.slice(0, p); + let after = self.slice(p + 1, self.native.len()); + Some((before, after)) + } + + pub fn split_at(&self, pos: usize) -> (Cow<'a, OsStr>, Cow<'a, OsStr>) { + let before = self.slice(0, pos); + let after = self.slice(pos, self.native.len()); + (before, after) + } + + pub fn strip_prefix(&self, prefix: &OsStr) -> Option> { + let n_prefix = to_native_int_representation(prefix); + let result = self.match_cow( + |b| b.strip_prefix(&*n_prefix).ok_or(()), + |o| o.strip_prefix(&*n_prefix).map(|x| x.to_vec()).ok_or(()), + ); + result.ok() + } + + pub fn strip_prefix_native(&self, prefix: &OsStr) -> Option> { + let n_prefix = to_native_int_representation(prefix); + let result = self.match_cow_native( + |b| b.strip_prefix(&*n_prefix).ok_or(()), + |o| o.strip_prefix(&*n_prefix).map(|x| x.to_vec()).ok_or(()), + ); + result.ok() + } + + fn match_cow( + &self, + f_borrow: FnBorrow, + f_owned: FnOwned, + ) -> Result, Err> + where + FnBorrow: FnOnce(&'a [NativeCharInt]) -> Result<&'a [NativeCharInt], Err>, + FnOwned: FnOnce(&Vec) -> Result, Err>, + { + match &self.native { + Cow::Borrowed(b) => { + let slice = f_borrow(b); + slice.map(|x| from_native_int_representation(Cow::Borrowed(x))) + } + Cow::Owned(o) => { + let slice = f_owned(o); + let os_str = slice.map(from_native_int_representation_owned); + os_str.map(Cow::Owned) + } + } + } + + fn match_cow_native( + &self, + f_borrow: FnBorrow, + f_owned: FnOwned, + ) -> Result, Err> + where + FnBorrow: FnOnce(&'a [NativeCharInt]) -> Result<&'a [NativeCharInt], Err>, + FnOwned: FnOnce(&Vec) -> Result, Err>, + { + match &self.native { + Cow::Borrowed(b) => { + let slice = f_borrow(b); + slice.map(Cow::Borrowed) + } + Cow::Owned(o) => { + let slice = f_owned(o); + slice.map(Cow::Owned) + } + } + } +} diff --git a/src/uu/env/src/split_iterator.rs b/src/uu/env/src/split_iterator.rs new file mode 100644 index 00000000000..40e5e9dae63 --- /dev/null +++ b/src/uu/env/src/split_iterator.rs @@ -0,0 +1,374 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// +// This file is based on work from Tomasz MiÄ…sko who published it as "shell_words" crate, +// licensed under the Apache License, Version 2.0 +// or the MIT license , at your option. +// +//! Process command line according to parsing rules of original GNU env. +//! Even though it looks quite like a POSIX syntax, the original +//! "shell_words" implementation had to be adapted significantly. +//! +//! Apart from the grammar differences, there is a new feature integrated: $VARIABLE expansion. +//! +//! [GNU env] +// spell-checker:ignore (words) Tomasz MiÄ…sko rntfv FFFD varname + +#![forbid(unsafe_code)] + +use std::borrow::Cow; + +use crate::EnvError; +use crate::native_int_str::NativeCharInt; +use crate::native_int_str::NativeIntStr; +use crate::native_int_str::NativeIntString; +use crate::native_int_str::from_native_int_representation; +use crate::string_expander::StringExpander; +use crate::string_parser::StringParser; +use crate::variable_parser::VariableParser; + +const BACKSLASH: char = '\\'; +const DOUBLE_QUOTES: char = '\"'; +const SINGLE_QUOTES: char = '\''; +const NEW_LINE: char = '\n'; +const DOLLAR: char = '$'; + +const REPLACEMENTS: [(char, char); 9] = [ + ('r', '\r'), + ('n', '\n'), + ('t', '\t'), + ('f', '\x0C'), + ('v', '\x0B'), + ('_', ' '), + ('#', '#'), + ('$', '$'), + ('"', '"'), +]; + +const ASCII_WHITESPACE_CHARS: [char; 6] = [' ', '\t', '\r', '\n', '\x0B', '\x0C']; + +pub struct SplitIterator<'a> { + expander: StringExpander<'a>, + words: Vec>, +} + +impl<'a> SplitIterator<'a> { + pub fn new(s: &'a NativeIntStr) -> Self { + Self { + expander: StringExpander::new(s), + words: Vec::new(), + } + } + + fn skip_one(&mut self) -> Result<(), EnvError> { + self.expander + .get_parser_mut() + .consume_one_ascii_or_all_non_ascii()?; + Ok(()) + } + + fn take_one(&mut self) -> Result<(), EnvError> { + Ok(self.expander.take_one()?) + } + + fn get_current_char(&self) -> Option { + self.expander.peek().ok() + } + + fn push_char_to_word(&mut self, c: char) { + self.expander.put_one_char(c); + } + + fn push_word_to_words(&mut self) { + let word = self.expander.take_collected_output(); + self.words.push(word); + } + + fn get_parser(&self) -> &StringParser<'a> { + self.expander.get_parser() + } + + fn get_parser_mut(&mut self) -> &mut StringParser<'a> { + self.expander.get_parser_mut() + } + + fn substitute_variable<'x>(&'x mut self) -> Result<(), EnvError> { + let mut var_parse = VariableParser::<'a, '_> { + parser: self.get_parser_mut(), + }; + + let (name, default) = var_parse.parse_variable()?; + + let varname_os_str_cow = from_native_int_representation(Cow::Borrowed(name)); + let value = std::env::var_os(varname_os_str_cow); + match (&value, default) { + (None, None) => {} // do nothing, just replace it with "" + (Some(value), _) => { + self.expander.put_string(value); + } + (None, Some(default)) => { + self.expander.put_native_string(default); + } + }; + + Ok(()) + } + + fn check_and_replace_ascii_escape_code(&mut self, c: char) -> Result { + if let Some(replace) = REPLACEMENTS.iter().find(|&x| x.0 == c) { + self.skip_one()?; + self.push_char_to_word(replace.1); + return Ok(true); + } + + Ok(false) + } + + fn make_invalid_sequence_backslash_xin_minus_s(&self, c: char) -> EnvError { + EnvError::EnvInvalidSequenceBackslashXInMinusS( + self.expander.get_parser().get_peek_position(), + c, + ) + } + + fn state_root(&mut self) -> Result<(), EnvError> { + loop { + match self.state_delimiter() { + Err(EnvError::EnvContinueWithDelimiter) => {} + Err(EnvError::EnvReachedEnd) => return Ok(()), + result => return result, + } + } + } + + fn state_delimiter(&mut self) -> Result<(), EnvError> { + loop { + match self.get_current_char() { + None => return Ok(()), + Some('#') => { + self.skip_one()?; + self.state_comment()?; + } + Some(BACKSLASH) => { + self.skip_one()?; + self.state_delimiter_backslash()?; + } + Some(c) if ASCII_WHITESPACE_CHARS.contains(&c) => { + self.skip_one()?; + } + Some(_) => { + // Don't consume char. Will be done in unquoted state. + self.state_unquoted()?; + } + } + } + } + + fn state_delimiter_backslash(&mut self) -> Result<(), EnvError> { + match self.get_current_char() { + None => Err(EnvError::EnvInvalidBackslashAtEndOfStringInMinusS( + self.get_parser().get_peek_position(), + "Delimiter".into(), + )), + Some('_' | NEW_LINE) => { + self.skip_one()?; + Ok(()) + } + Some(DOLLAR | BACKSLASH | '#' | SINGLE_QUOTES | DOUBLE_QUOTES) => { + self.take_one()?; + self.state_unquoted() + } + Some('c') => Err(EnvError::EnvReachedEnd), + Some(c) if self.check_and_replace_ascii_escape_code(c)? => self.state_unquoted(), + Some(c) => Err(self.make_invalid_sequence_backslash_xin_minus_s(c)), + } + } + + fn state_unquoted(&mut self) -> Result<(), EnvError> { + loop { + match self.get_current_char() { + None => { + self.push_word_to_words(); + return Err(EnvError::EnvReachedEnd); + } + Some(DOLLAR) => { + self.substitute_variable()?; + } + Some(SINGLE_QUOTES) => { + self.skip_one()?; + self.state_single_quoted()?; + } + Some(DOUBLE_QUOTES) => { + self.skip_one()?; + self.state_double_quoted()?; + } + Some(BACKSLASH) => { + self.skip_one()?; + self.state_unquoted_backslash()?; + } + Some(c) if ASCII_WHITESPACE_CHARS.contains(&c) => { + self.push_word_to_words(); + self.skip_one()?; + return Ok(()); + } + Some(_) => { + self.take_one()?; + } + } + } + } + + fn state_unquoted_backslash(&mut self) -> Result<(), EnvError> { + match self.get_current_char() { + None => Err(EnvError::EnvInvalidBackslashAtEndOfStringInMinusS( + self.get_parser().get_peek_position(), + "Unquoted".into(), + )), + Some(NEW_LINE) => { + self.skip_one()?; + Ok(()) + } + Some('_') => { + self.skip_one()?; + self.push_word_to_words(); + Err(EnvError::EnvContinueWithDelimiter) + } + Some('c') => { + self.push_word_to_words(); + Err(EnvError::EnvReachedEnd) + } + Some(DOLLAR | BACKSLASH | SINGLE_QUOTES | DOUBLE_QUOTES) => { + self.take_one()?; + Ok(()) + } + Some(c) if self.check_and_replace_ascii_escape_code(c)? => Ok(()), + Some(c) => Err(self.make_invalid_sequence_backslash_xin_minus_s(c)), + } + } + + fn state_single_quoted(&mut self) -> Result<(), EnvError> { + loop { + match self.get_current_char() { + None => { + return Err(EnvError::EnvMissingClosingQuote( + self.get_parser().get_peek_position(), + '\'', + )); + } + Some(SINGLE_QUOTES) => { + self.skip_one()?; + return Ok(()); + } + Some(BACKSLASH) => { + self.skip_one()?; + self.split_single_quoted_backslash()?; + } + Some(_) => { + self.take_one()?; + } + } + } + } + + fn split_single_quoted_backslash(&mut self) -> Result<(), EnvError> { + match self.get_current_char() { + None => Err(EnvError::EnvMissingClosingQuote( + self.get_parser().get_peek_position(), + '\'', + )), + Some(NEW_LINE) => { + self.skip_one()?; + Ok(()) + } + Some(SINGLE_QUOTES | BACKSLASH) => { + self.take_one()?; + Ok(()) + } + Some(c) if REPLACEMENTS.iter().any(|&x| x.0 == c) => { + // See GNU test-suite e11: In single quotes, \t remains as it is. + // Comparing with GNU behavior: \a is not accepted and issues an error. + // So apparently only known sequences are allowed, even though they are not expanded.... bug of GNU? + self.push_char_to_word(BACKSLASH); + self.take_one()?; + Ok(()) + } + Some(c) => Err(self.make_invalid_sequence_backslash_xin_minus_s(c)), + } + } + + fn state_double_quoted(&mut self) -> Result<(), EnvError> { + loop { + match self.get_current_char() { + None => { + return Err(EnvError::EnvMissingClosingQuote( + self.get_parser().get_peek_position(), + '"', + )); + } + Some(DOLLAR) => { + self.substitute_variable()?; + } + Some(DOUBLE_QUOTES) => { + self.skip_one()?; + return Ok(()); + } + Some(BACKSLASH) => { + self.skip_one()?; + self.state_double_quoted_backslash()?; + } + Some(_) => { + self.take_one()?; + } + } + } + } + + fn state_double_quoted_backslash(&mut self) -> Result<(), EnvError> { + match self.get_current_char() { + None => Err(EnvError::EnvMissingClosingQuote( + self.get_parser().get_peek_position(), + '"', + )), + Some(NEW_LINE) => { + self.skip_one()?; + Ok(()) + } + Some(DOUBLE_QUOTES | DOLLAR | BACKSLASH) => { + self.take_one()?; + Ok(()) + } + Some('c') => Err(EnvError::EnvBackslashCNotAllowedInDoubleQuotes( + self.get_parser().get_peek_position(), + )), + Some(c) if self.check_and_replace_ascii_escape_code(c)? => Ok(()), + Some(c) => Err(self.make_invalid_sequence_backslash_xin_minus_s(c)), + } + } + + fn state_comment(&mut self) -> Result<(), EnvError> { + loop { + match self.get_current_char() { + None => return Err(EnvError::EnvReachedEnd), + Some(NEW_LINE) => { + self.skip_one()?; + return Ok(()); + } + Some(_) => { + self.get_parser_mut().skip_until_char_or_end(NEW_LINE); + } + } + } + } + + pub fn split(mut self) -> Result, EnvError> { + self.state_root()?; + Ok(self.words) + } +} + +pub fn split(s: &NativeIntStr) -> Result, EnvError> { + let split_args = SplitIterator::new(s).split()?; + Ok(split_args) +} diff --git a/src/uu/env/src/string_expander.rs b/src/uu/env/src/string_expander.rs new file mode 100644 index 00000000000..48f98e1fe6f --- /dev/null +++ b/src/uu/env/src/string_expander.rs @@ -0,0 +1,91 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::{ + ffi::{OsStr, OsString}, + mem, +}; + +use crate::{ + native_int_str::{NativeCharInt, NativeIntStr, to_native_int_representation}, + string_parser::{Chunk, Error, StringParser}, +}; + +/// This class makes parsing and word collection more convenient. +/// +/// It manages an "output" buffer that is automatically filled. +/// It provides "skip_one" and "take_one" that focus on +/// working with ASCII separators. Thus they will skip or take +/// all consecutive non-ascii char sequences at once. +pub struct StringExpander<'a> { + parser: StringParser<'a>, + output: Vec, +} + +impl<'a> StringExpander<'a> { + pub fn new(input: &'a NativeIntStr) -> Self { + Self { + parser: StringParser::new(input), + output: Vec::default(), + } + } + + pub fn new_at(input: &'a NativeIntStr, pos: usize) -> Self { + Self { + parser: StringParser::new_at(input, pos), + output: Vec::default(), + } + } + + pub fn get_parser(&self) -> &StringParser<'a> { + &self.parser + } + + pub fn get_parser_mut(&mut self) -> &mut StringParser<'a> { + &mut self.parser + } + + pub fn peek(&self) -> Result { + self.parser.peek() + } + + pub fn skip_one(&mut self) -> Result<(), Error> { + self.get_parser_mut().consume_one_ascii_or_all_non_ascii()?; + Ok(()) + } + + pub fn get_peek_position(&self) -> usize { + self.get_parser().get_peek_position() + } + + pub fn take_one(&mut self) -> Result<(), Error> { + let chunks = self.parser.consume_one_ascii_or_all_non_ascii()?; + for chunk in chunks { + match chunk { + Chunk::InvalidEncoding(invalid) => self.output.extend(invalid), + Chunk::ValidSingleIntChar((_c, ni)) => self.output.push(ni), + } + } + Ok(()) + } + + pub fn put_one_char(&mut self, c: char) { + let os_str = OsString::from(c.to_string()); + self.put_string(os_str); + } + + pub fn put_string>(&mut self, os_str: S) { + let native = to_native_int_representation(os_str.as_ref()); + self.output.extend(&*native); + } + + pub fn put_native_string(&mut self, n_str: &NativeIntStr) { + self.output.extend(n_str); + } + + pub fn take_collected_output(&mut self) -> Vec { + mem::take(&mut self.output) + } +} diff --git a/src/uu/env/src/string_parser.rs b/src/uu/env/src/string_parser.rs new file mode 100644 index 00000000000..84eb346fa4c --- /dev/null +++ b/src/uu/env/src/string_parser.rs @@ -0,0 +1,181 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// +// spell-checker:ignore (words) FFFD +#![forbid(unsafe_code)] + +use std::{borrow::Cow, ffi::OsStr}; + +use crate::native_int_str::{ + NativeCharInt, NativeIntStr, from_native_int_representation, get_char_from_native_int, + get_single_native_int_value, +}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Error { + pub peek_position: usize, + pub err_type: ErrorType, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ErrorType { + EndOfInput, + InternalError, +} + +/// Provides a valid char or a invalid sequence of bytes. +/// +/// Invalid byte sequences can't be split in any meaningful way. +/// Thus, they need to be consumed as one piece. +pub enum Chunk<'a> { + InvalidEncoding(&'a NativeIntStr), + ValidSingleIntChar((char, NativeCharInt)), +} + +/// This class makes parsing a OsString char by char more convenient. +/// +/// It also allows to capturing of intermediate positions for later splitting. +pub struct StringParser<'a> { + input: &'a NativeIntStr, + pointer: usize, + remaining: &'a NativeIntStr, +} + +impl<'a> StringParser<'a> { + pub fn new(input: &'a NativeIntStr) -> Self { + let mut instance = Self { + input, + pointer: 0, + remaining: input, + }; + instance.set_pointer(0); + instance + } + + pub fn new_at(input: &'a NativeIntStr, pos: usize) -> Self { + let mut instance = Self::new(input); + instance.set_pointer(pos); + instance + } + + pub fn get_input(&self) -> &'a NativeIntStr { + self.input + } + + pub fn get_peek_position(&self) -> usize { + self.pointer + } + + pub fn peek(&self) -> Result { + self.peek_char_at_pointer(self.pointer) + } + + fn make_err(&self, err_type: ErrorType) -> Error { + Error { + peek_position: self.get_peek_position(), + err_type, + } + } + + pub fn peek_char_at_pointer(&self, at_pointer: usize) -> Result { + let split = self.input.split_at(at_pointer).1; + if split.is_empty() { + return Err(self.make_err(ErrorType::EndOfInput)); + } + if let Some((c, _ni)) = get_char_from_native_int(split[0]) { + Ok(c) + } else { + Ok('\u{FFFD}') + } + } + + fn get_chunk_with_length_at(&self, pointer: usize) -> Result<(Chunk<'a>, usize), Error> { + let (_before, after) = self.input.split_at(pointer); + if after.is_empty() { + return Err(self.make_err(ErrorType::EndOfInput)); + } + + if let Some(c_ni) = get_char_from_native_int(after[0]) { + Ok((Chunk::ValidSingleIntChar(c_ni), 1)) + } else { + let mut i = 1; + while i < after.len() { + if let Some(_c) = get_char_from_native_int(after[i]) { + break; + } + i += 1; + } + + let chunk = &after[0..i]; + Ok((Chunk::InvalidEncoding(chunk), chunk.len())) + } + } + + pub fn peek_chunk(&self) -> Option> { + self.get_chunk_with_length_at(self.pointer) + .ok() + .map(|(chunk, _)| chunk) + } + + pub fn consume_chunk(&mut self) -> Result, Error> { + let (chunk, len) = self.get_chunk_with_length_at(self.pointer)?; + self.set_pointer(self.pointer + len); + Ok(chunk) + } + + pub fn consume_one_ascii_or_all_non_ascii(&mut self) -> Result>, Error> { + let mut result = Vec::>::new(); + loop { + let data = self.consume_chunk()?; + let was_ascii = if let Chunk::ValidSingleIntChar((c, _ni)) = &data { + c.is_ascii() + } else { + false + }; + result.push(data); + if was_ascii { + return Ok(result); + } + + match self.peek_chunk() { + Some(Chunk::ValidSingleIntChar((c, _ni))) if c.is_ascii() => return Ok(result), + None => return Ok(result), + _ => {} + } + } + } + + pub fn skip_multiple(&mut self, skip_byte_count: usize) { + let end_ptr = self.pointer + skip_byte_count; + self.set_pointer(end_ptr); + } + + pub fn skip_until_char_or_end(&mut self, c: char) { + let native_rep = get_single_native_int_value(&c).unwrap(); + let pos = self.remaining.iter().position(|x| *x == native_rep); + + if let Some(pos) = pos { + self.set_pointer(self.pointer + pos); + } else { + self.set_pointer(self.input.len()); + } + } + + pub fn substring(&self, range: &std::ops::Range) -> &'a NativeIntStr { + let (_before1, after1) = self.input.split_at(range.start); + let (middle, _after2) = after1.split_at(range.end - range.start); + middle + } + + pub fn peek_remaining(&self) -> Cow<'a, OsStr> { + from_native_int_representation(Cow::Borrowed(self.remaining)) + } + + pub fn set_pointer(&mut self, new_pointer: usize) { + self.pointer = new_pointer; + let (_before, after) = self.input.split_at(self.pointer); + self.remaining = after; + } +} diff --git a/src/uu/env/src/variable_parser.rs b/src/uu/env/src/variable_parser.rs new file mode 100644 index 00000000000..2555c709f43 --- /dev/null +++ b/src/uu/env/src/variable_parser.rs @@ -0,0 +1,159 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::ops::Range; + +use crate::EnvError; +use crate::{native_int_str::NativeIntStr, string_parser::StringParser}; + +pub struct VariableParser<'a, 'b> { + pub parser: &'b mut StringParser<'a>, +} + +impl<'a> VariableParser<'a, '_> { + fn get_current_char(&self) -> Option { + self.parser.peek().ok() + } + + fn check_variable_name_start(&self) -> Result<(), EnvError> { + if let Some(c) = self.get_current_char() { + if c.is_ascii_digit() { + return Err(EnvError::EnvParsingOfVariableUnexpectedNumber( + self.parser.get_peek_position(), + c.to_string(), + )); + } + } + Ok(()) + } + + fn skip_one(&mut self) -> Result<(), EnvError> { + self.parser.consume_chunk()?; + Ok(()) + } + + fn parse_braced_variable_name( + &mut self, + ) -> Result<(&'a NativeIntStr, Option<&'a NativeIntStr>), EnvError> { + let pos_start = self.parser.get_peek_position(); + + self.check_variable_name_start()?; + + let (varname_end, default_end); + loop { + match self.get_current_char() { + None => { + return Err(EnvError::EnvParsingOfVariableMissingClosingBrace( + self.parser.get_peek_position(), + )); + } + Some(c) if !c.is_ascii() || c.is_ascii_alphanumeric() || c == '_' => { + self.skip_one()?; + } + Some(':') => { + varname_end = self.parser.get_peek_position(); + loop { + match self.get_current_char() { + None => { + return Err( + EnvError::EnvParsingOfVariableMissingClosingBraceAfterValue( + self.parser.get_peek_position(), + ), + ); + } + Some('}') => { + default_end = Some(self.parser.get_peek_position()); + self.skip_one()?; + break; + } + Some(_) => { + self.skip_one()?; + } + } + } + break; + } + Some('}') => { + varname_end = self.parser.get_peek_position(); + default_end = None; + self.skip_one()?; + break; + } + Some(c) => { + return Err(EnvError::EnvParsingOfVariableExceptedBraceOrColon( + self.parser.get_peek_position(), + c.to_string(), + )); + } + }; + } + + let default_opt = if let Some(default_end) = default_end { + Some(self.parser.substring(&Range { + start: varname_end + 1, + end: default_end, + })) + } else { + None + }; + + let varname = self.parser.substring(&Range { + start: pos_start, + end: varname_end, + }); + + Ok((varname, default_opt)) + } + + fn parse_unbraced_variable_name(&mut self) -> Result<&'a NativeIntStr, EnvError> { + let pos_start = self.parser.get_peek_position(); + + self.check_variable_name_start()?; + + loop { + match self.get_current_char() { + None => break, + Some(c) if c.is_ascii_alphanumeric() || c == '_' => { + self.skip_one()?; + } + Some(_) => break, + }; + } + + let pos_end = self.parser.get_peek_position(); + + if pos_end == pos_start { + return Err(EnvError::EnvParsingOfMissingVariable(pos_start)); + } + + let varname = self.parser.substring(&Range { + start: pos_start, + end: pos_end, + }); + + Ok(varname) + } + + pub fn parse_variable( + &mut self, + ) -> Result<(&'a NativeIntStr, Option<&'a NativeIntStr>), EnvError> { + self.skip_one()?; + + let (name, default) = match self.get_current_char() { + None => { + return Err(EnvError::EnvParsingOfMissingVariable( + self.parser.get_peek_position(), + )); + } + Some('{') => { + self.skip_one()?; + self.parse_braced_variable_name()? + } + Some(_) => (self.parse_unbraced_variable_name()?, None), + }; + + Ok((name, default)) + } +} diff --git a/src/uu/expand/Cargo.toml b/src/uu/expand/Cargo.toml index 87f8f8f7716..cd96b0278f7 100644 --- a/src/uu/expand/Cargo.toml +++ b/src/uu/expand/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_expand" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "expand ~ (uutils) convert input tabs to spaces" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/expand" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/expand.rs" @@ -18,6 +21,7 @@ path = "src/expand.rs" clap = { workspace = true } unicode-width = { workspace = true } uucore = { workspace = true } +thiserror = { workspace = true } [[bin]] name = "expand" diff --git a/src/uu/expand/src/expand.rs b/src/uu/expand/src/expand.rs index 1efb36c654c..4c37393b4ac 100644 --- a/src/uu/expand/src/expand.rs +++ b/src/uu/expand/src/expand.rs @@ -5,17 +5,17 @@ // spell-checker:ignore (ToDO) ctype cwidth iflag nbytes nspaces nums tspaces uflag Preprocess -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; -use std::error::Error; -use std::fmt; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use std::ffi::OsString; use std::fs::File; -use std::io::{stdin, stdout, BufRead, BufReader, BufWriter, Read, Write}; +use std::io::{BufRead, BufReader, BufWriter, Read, Write, stdin, stdout}; use std::num::IntErrorKind; use std::path::Path; use std::str::from_utf8; +use thiserror::Error; use unicode_width::UnicodeWidthChar; use uucore::display::Quotable; -use uucore::error::{set_exit_code, FromIo, UError, UResult}; +use uucore::error::{FromIo, UError, UResult, set_exit_code}; use uucore::{format_usage, help_about, help_usage, show_error}; const ABOUT: &str = help_about!("expand.md"); @@ -60,43 +60,24 @@ fn is_digit_or_comma(c: char) -> bool { } /// Errors that can occur when parsing a `--tabs` argument. -#[derive(Debug)] +#[derive(Debug, Error)] enum ParseError { + #[error("tab size contains invalid character(s): {}", .0.quote())] InvalidCharacter(String), + #[error("{} specifier not at start of number: {}", .0.quote(), .1.quote())] SpecifierNotAtStartOfNumber(String, String), + #[error("{} specifier only allowed with the last value", .0.quote())] SpecifierOnlyAllowedWithLastValue(String), + #[error("tab size cannot be 0")] TabSizeCannotBeZero, + #[error("tab stop is too large {}", .0.quote())] TabSizeTooLarge(String), + #[error("tab sizes must be ascending")] TabSizesMustBeAscending, } -impl Error for ParseError {} impl UError for ParseError {} -impl fmt::Display for ParseError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::InvalidCharacter(s) => { - write!(f, "tab size contains invalid character(s): {}", s.quote()) - } - Self::SpecifierNotAtStartOfNumber(specifier, s) => write!( - f, - "{} specifier not at start of number: {}", - specifier.quote(), - s.quote(), - ), - Self::SpecifierOnlyAllowedWithLastValue(specifier) => write!( - f, - "{} specifier only allowed with the last value", - specifier.quote() - ), - Self::TabSizeCannotBeZero => write!(f, "tab size cannot be 0"), - Self::TabSizeTooLarge(s) => write!(f, "tab stop is too large {}", s.quote()), - Self::TabSizesMustBeAscending => write!(f, "tab sizes must be ascending"), - } - } -} - /// Parse a list of tabstops from a `--tabs` argument. /// /// This function returns both the vector of numbers appearing in the @@ -164,14 +145,14 @@ fn tabstops_parse(s: &str) -> Result<(RemainingMode, Vec), ParseError> { } let s = s.trim_start_matches(char::is_numeric); - if s.starts_with('/') || s.starts_with('+') { - return Err(ParseError::SpecifierNotAtStartOfNumber( + return if s.starts_with('/') || s.starts_with('+') { + Err(ParseError::SpecifierNotAtStartOfNumber( s[0..1].to_string(), s.to_string(), - )); + )) } else { - return Err(ParseError::InvalidCharacter(s.to_string())); - } + Err(ParseError::InvalidCharacter(s.to_string())) + }; } } } @@ -243,18 +224,20 @@ impl Options { /// Preprocess command line arguments and expand shortcuts. For example, "-7" is expanded to /// "--tabs=7" and "-1,3" to "--tabs=1 --tabs=3". -fn expand_shortcuts(args: &[String]) -> Vec { +fn expand_shortcuts(args: Vec) -> Vec { let mut processed_args = Vec::with_capacity(args.len()); for arg in args { - if arg.starts_with('-') && arg[1..].chars().all(is_digit_or_comma) { - arg[1..] - .split(',') - .filter(|s| !s.is_empty()) - .for_each(|s| processed_args.push(format!("--tabs={s}"))); - } else { - processed_args.push(arg.to_string()); + if let Some(arg) = arg.to_str() { + if arg.starts_with('-') && arg[1..].chars().all(is_digit_or_comma) { + arg[1..] + .split(',') + .filter(|s| !s.is_empty()) + .for_each(|s| processed_args.push(OsString::from(format!("--tabs={s}")))); + continue; + } } + processed_args.push(arg); } processed_args @@ -262,16 +245,14 @@ fn expand_shortcuts(args: &[String]) -> Vec { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_ignore(); - - let matches = uu_app().try_get_matches_from(expand_shortcuts(&args))?; + let matches = uu_app().try_get_matches_from(expand_shortcuts(args.collect()))?; expand(&Options::new(&matches)?) } pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .after_help(LONG_HELP) .override_usage(format_usage(USAGE)) @@ -375,7 +356,7 @@ fn expand_line( tabstops: &[usize], options: &Options, ) -> std::io::Result<()> { - use self::CharType::*; + use self::CharType::{Backspace, Other, Tab}; let mut col = 0; let mut byte = 0; @@ -424,7 +405,7 @@ fn expand_line( // now dump out either spaces if we're expanding, or a literal tab if we're not if init || !options.iflag { if nts <= options.tspaces.len() { - output.write_all(options.tspaces[..nts].as_bytes())?; + output.write_all(&options.tspaces.as_bytes()[..nts])?; } else { output.write_all(" ".repeat(nts).as_bytes())?; }; @@ -467,19 +448,25 @@ fn expand(options: &Options) -> UResult<()> { for file in &options.files { if Path::new(file).is_dir() { - show_error!("{}: Is a directory", file); + show_error!("{file}: Is a directory"); set_exit_code(1); continue; } - - let mut fh = open(file)?; - - while match fh.read_until(b'\n', &mut buf) { - Ok(s) => s > 0, - Err(_) => buf.is_empty(), - } { - expand_line(&mut buf, &mut output, ts, options) - .map_err_context(|| "failed to write output".to_string())?; + match open(file) { + Ok(mut fh) => { + while match fh.read_until(b'\n', &mut buf) { + Ok(s) => s > 0, + Err(_) => buf.is_empty(), + } { + expand_line(&mut buf, &mut output, ts, options) + .map_err_context(|| "failed to write output".to_string())?; + } + } + Err(e) => { + show_error!("{e}"); + set_exit_code(1); + continue; + } } } Ok(()) @@ -489,8 +476,8 @@ fn expand(options: &Options) -> UResult<()> { mod tests { use crate::is_digit_or_comma; - use super::next_tabstop; use super::RemainingMode; + use super::next_tabstop; #[test] fn test_next_tabstop_remaining_mode_none() { diff --git a/src/uu/expr/Cargo.toml b/src/uu/expr/Cargo.toml index 17d37413e26..00e3e3cab03 100644 --- a/src/uu/expr/Cargo.toml +++ b/src/uu/expr/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_expr" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "expr ~ (uutils) display the value of EXPRESSION" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/expr" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/expr.rs" @@ -20,6 +23,7 @@ num-bigint = { workspace = true } num-traits = { workspace = true } onig = { workspace = true } uucore = { workspace = true } +thiserror = { workspace = true } [[bin]] name = "expr" diff --git a/src/uu/expr/src/expr.rs b/src/uu/expr/src/expr.rs index 4e41a6929e6..073bf501a0b 100644 --- a/src/uu/expr/src/expr.rs +++ b/src/uu/expr/src/expr.rs @@ -3,18 +3,15 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -use std::fmt::Display; - -use clap::{crate_version, Arg, ArgAction, Command}; -use syntax_tree::AstNode; +use clap::{Arg, ArgAction, Command}; +use syntax_tree::{AstNode, is_truthy}; +use thiserror::Error; use uucore::{ display::Quotable, error::{UError, UResult}, format_usage, help_about, help_section, help_usage, }; -use crate::syntax_tree::is_truthy; - mod syntax_tree; mod options { @@ -25,39 +22,36 @@ mod options { pub type ExprResult = Result; -#[derive(Debug, PartialEq, Eq)] +#[derive(Error, Clone, Debug, PartialEq, Eq)] pub enum ExprError { + #[error("syntax error: unexpected argument {}", .0.quote())] UnexpectedArgument(String), + #[error("syntax error: missing argument after {}", .0.quote())] MissingArgument(String), + #[error("non-integer argument")] NonIntegerArgument, + #[error("missing operand")] MissingOperand, + #[error("division by zero")] DivisionByZero, + #[error("Invalid regex expression")] InvalidRegexExpression, + #[error("syntax error: expecting ')' after {}", .0.quote())] ExpectedClosingBraceAfter(String), + #[error("syntax error: expecting ')' instead of {}", .0.quote())] + ExpectedClosingBraceInsteadOf(String), + #[error("Unmatched ( or \\(")] + UnmatchedOpeningParenthesis, + #[error("Unmatched ) or \\)")] + UnmatchedClosingParenthesis, + #[error("Unmatched \\{{")] + UnmatchedOpeningBrace, + #[error("Unmatched ) or \\}}")] + UnmatchedClosingBrace, + #[error("Invalid content of \\{{\\}}")] + InvalidBracketContent, } -impl Display for ExprError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::UnexpectedArgument(s) => { - write!(f, "syntax error: unexpected argument {}", s.quote()) - } - Self::MissingArgument(s) => { - write!(f, "syntax error: missing argument after {}", s.quote()) - } - Self::NonIntegerArgument => write!(f, "non-integer argument"), - Self::MissingOperand => write!(f, "missing operand"), - Self::DivisionByZero => write!(f, "division by zero"), - Self::InvalidRegexExpression => write!(f, "Invalid regex expression"), - Self::ExpectedClosingBraceAfter(s) => { - write!(f, "expected ')' after {}", s.quote()) - } - } - } -} - -impl std::error::Error for ExprError {} - impl UError for ExprError { fn code(&self) -> i32 { 2 @@ -70,7 +64,7 @@ impl UError for ExprError { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(help_about!("expr.md")) .override_usage(format_usage(help_usage!("expr.md"))) .after_help(help_section!("after help", "expr.md")) @@ -100,16 +94,29 @@ pub fn uu_app() -> Command { pub fn uumain(args: impl uucore::Args) -> UResult<()> { // For expr utility we do not want getopts. // The following usage should work without escaping hyphens: `expr -15 = 1 + 2 \* \( 3 - -4 \)` - let matches = uu_app().try_get_matches_from(args)?; - let token_strings: Vec<&str> = matches - .get_many::(options::EXPRESSION) - .map(|v| v.into_iter().map(|s| s.as_ref()).collect::>()) - .unwrap_or_default(); + let args: Vec = args + .skip(1) // Skip binary name + .map(|a| a.to_string_lossy().to_string()) + .collect(); + + if args.len() == 1 && args[0] == "--help" { + let _ = uu_app().print_help(); + } else if args.len() == 1 && args[0] == "--version" { + println!("{} {}", uucore::util_name(), uucore::crate_version!()); + } else { + // The first argument may be "--" and should be be ignored. + let args = if !args.is_empty() && args[0] == "--" { + &args[1..] + } else { + &args + }; - let res: String = AstNode::parse(&token_strings)?.eval()?.eval_as_string(); - println!("{res}"); - if !is_truthy(&res.into()) { - return Err(1.into()); + let res: String = AstNode::parse(args)?.eval()?.eval_as_string(); + println!("{res}"); + if !is_truthy(&res.into()) { + return Err(1.into()); + } } + Ok(()) } diff --git a/src/uu/expr/src/syntax_tree.rs b/src/uu/expr/src/syntax_tree.rs index 5aa9c93986b..3026d5d41b4 100644 --- a/src/uu/expr/src/syntax_tree.rs +++ b/src/uu/expr/src/syntax_tree.rs @@ -5,8 +5,10 @@ // spell-checker:ignore (ToDO) ints paren prec multibytes +use std::{cell::Cell, collections::BTreeMap}; + use num_bigint::{BigInt, ParseBigIntError}; -use num_traits::ToPrimitive; +use num_traits::{ToPrimitive, Zero}; use onig::{Regex, RegexOptions, Syntax}; use crate::{ExprError, ExprResult}; @@ -46,7 +48,11 @@ pub enum StringOp { } impl BinOp { - fn eval(&self, left: &AstNode, right: &AstNode) -> ExprResult { + fn eval( + &self, + left: ExprResult, + right: ExprResult, + ) -> ExprResult { match self { Self::Relation(op) => op.eval(left, right), Self::Numeric(op) => op.eval(left, right), @@ -56,9 +62,9 @@ impl BinOp { } impl RelationOp { - fn eval(&self, a: &AstNode, b: &AstNode) -> ExprResult { - let a = a.eval()?; - let b = b.eval()?; + fn eval(&self, a: ExprResult, b: ExprResult) -> ExprResult { + let a = a?; + let b = b?; let b = if let (Ok(a), Ok(b)) = (&a.to_bigint(), &b.to_bigint()) { match self { Self::Lt => a < b, @@ -81,18 +87,18 @@ impl RelationOp { Self::Geq => a >= b, } }; - if b { - Ok(1.into()) - } else { - Ok(0.into()) - } + if b { Ok(1.into()) } else { Ok(0.into()) } } } impl NumericOp { - fn eval(&self, left: &AstNode, right: &AstNode) -> ExprResult { - let a = left.eval()?.eval_as_bigint()?; - let b = right.eval()?.eval_as_bigint()?; + fn eval( + &self, + left: ExprResult, + right: ExprResult, + ) -> ExprResult { + let a = left?.eval_as_bigint()?; + let b = right?.eval_as_bigint()?; Ok(NumOrStr::Num(match self { Self::Add => a + b, Self::Sub => a - b, @@ -112,43 +118,79 @@ impl NumericOp { } impl StringOp { - fn eval(&self, left: &AstNode, right: &AstNode) -> ExprResult { + fn eval( + &self, + left: ExprResult, + right: ExprResult, + ) -> ExprResult { match self { Self::Or => { - let left = left.eval()?; + let left = left?; if is_truthy(&left) { return Ok(left); } - let right = right.eval()?; + let right = right?; if is_truthy(&right) { return Ok(right); } Ok(0.into()) } Self::And => { - let left = left.eval()?; + let left = left?; if !is_truthy(&left) { return Ok(0.into()); } - let right = right.eval()?; + let right = right?; if !is_truthy(&right) { return Ok(0.into()); } Ok(left) } Self::Match => { - let left = left.eval()?.eval_as_string(); - let right = right.eval()?.eval_as_string(); - let re_string = format!("^{}", right); + let left = left?.eval_as_string(); + let right = right?.eval_as_string(); + check_posix_regex_errors(&right)?; + + // All patterns are anchored so they begin with a caret (^) + let mut re_string = String::with_capacity(right.len() + 1); + re_string.push('^'); + + // Handle first character from the input pattern + let mut pattern_chars = right.chars(); + let first = pattern_chars.next(); + match first { + Some('^') => {} // Start of string anchor is already added + Some('*') => re_string.push_str(r"\*"), + Some(char) => re_string.push(char), + None => return Ok(0.into()), + }; + + // Handle the rest of the input pattern. + // Escaped previous character should not affect the current. + let mut prev = first.unwrap_or_default(); + let mut prev_is_escaped = false; + for curr in pattern_chars { + match curr { + // Carets are interpreted literally, unless used as character class negation "[^a]" + '^' if prev_is_escaped || !matches!(prev, '\\' | '[') => { + re_string.push_str(r"\^"); + } + char => re_string.push(char), + } + + prev_is_escaped = prev == '\\' && !prev_is_escaped; + prev = curr; + } + let re = Regex::with_options( &re_string, - RegexOptions::REGEX_OPTION_NONE, + RegexOptions::REGEX_OPTION_SINGLELINE, Syntax::grep(), ) .map_err(|_| ExprError::InvalidRegexExpression)?; Ok(if re.captures_len() > 0 { re.captures(&left) - .map(|captures| captures.at(1).unwrap()) + .and_then(|captures| captures.at(1)) .unwrap_or("") .to_string() } else { @@ -158,8 +200,8 @@ impl StringOp { .into()) } Self::Index => { - let left = left.eval()?.eval_as_string(); - let right = right.eval()?.eval_as_string(); + let left = left?.eval_as_string(); + let right = right?.eval_as_string(); for (current_idx, ch_h) in left.chars().enumerate() { for ch_n in right.to_string().chars() { if ch_n == ch_h { @@ -173,6 +215,99 @@ impl StringOp { } } +/// Check for errors in a supplied regular expression +/// +/// GNU coreutils shows messages for invalid regular expressions +/// differently from the oniguruma library used by the regex crate. +/// This method attempts to do these checks manually in one pass +/// through the regular expression. +/// +/// This method is not comprehensively checking all cases in which +/// a regular expression could be invalid; any cases not caught will +/// result in a [ExprError::InvalidRegexExpression] when passing the +/// regular expression through the Oniguruma bindings. This method is +/// intended to just identify a few situations for which GNU coreutils +/// has specific error messages. +fn check_posix_regex_errors(pattern: &str) -> ExprResult<()> { + let mut escaped_parens: u64 = 0; + let mut escaped_braces: u64 = 0; + let mut escaped = false; + + let mut repeating_pattern_text = String::new(); + let mut invalid_content_error = false; + + for c in pattern.chars() { + match (escaped, c) { + (true, ')') => { + escaped_parens = escaped_parens + .checked_sub(1) + .ok_or(ExprError::UnmatchedClosingParenthesis)?; + } + (true, '(') => { + escaped_parens += 1; + } + (true, '}') => { + escaped_braces = escaped_braces + .checked_sub(1) + .ok_or(ExprError::UnmatchedClosingBrace)?; + let mut repetition = + repeating_pattern_text[..repeating_pattern_text.len() - 1].splitn(2, ','); + match ( + repetition + .next() + .expect("splitn always returns at least one string"), + repetition.next(), + ) { + ("", None) => { + // Empty repeating pattern + invalid_content_error = true; + } + (x, None | Some("")) => { + if x.parse::().is_err() { + invalid_content_error = true; + } + } + ("", Some(x)) => { + if x.parse::().is_err() { + invalid_content_error = true; + } + } + (f, Some(l)) => { + if let (Ok(f), Ok(l)) = (f.parse::(), l.parse::()) { + invalid_content_error = invalid_content_error || f > l; + } else { + invalid_content_error = true; + } + } + } + repeating_pattern_text.clear(); + } + (true, '{') => { + escaped_braces += 1; + } + _ => { + if escaped_braces > 0 && repeating_pattern_text.len() <= 13 { + repeating_pattern_text.push(c); + } + if escaped_braces > 0 && !(c.is_ascii_digit() || c == '\\' || c == ',') { + invalid_content_error = true; + } + } + } + escaped = !escaped && c == '\\'; + } + match ( + escaped_parens.is_zero(), + escaped_braces.is_zero(), + invalid_content_error, + ) { + (true, true, false) => Ok(()), + (_, false, _) => Err(ExprError::UnmatchedOpeningBrace), + (false, _, _) => Err(ExprError::UnmatchedOpeningParenthesis), + (true, true, true) => Err(ExprError::InvalidBracketContent), + } +} + /// Precedence for infix binary operators const PRECEDENCE: &[&[(&str, BinOp)]] = &[ &[("|", BinOp::String(StringOp::Or))], @@ -197,7 +332,7 @@ const PRECEDENCE: &[&[(&str, BinOp)]] = &[ &[(":", BinOp::String(StringOp::Match))], ]; -#[derive(Debug)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum NumOrStr { Num(BigInt), Str(String), @@ -246,8 +381,19 @@ impl NumOrStr { } } -#[derive(Debug, PartialEq, Eq)] -pub enum AstNode { +#[derive(Debug, Clone)] +pub struct AstNode { + id: u32, + inner: AstNodeInner, +} + +// We derive Eq and PartialEq only for tests because we want to ignore the id field. +#[derive(Debug, Clone)] +#[cfg_attr(test, derive(Eq, PartialEq))] +pub enum AstNodeInner { + Evaluated { + value: NumOrStr, + }, Leaf { value: String, }, @@ -267,68 +413,139 @@ pub enum AstNode { } impl AstNode { - pub fn parse(input: &[&str]) -> ExprResult { + pub fn parse(input: &[impl AsRef]) -> ExprResult { Parser::new(input).parse() } + pub fn evaluated(self) -> ExprResult { + Ok(Self { + id: get_next_id(), + inner: AstNodeInner::Evaluated { + value: self.eval()?, + }, + }) + } + pub fn eval(&self) -> ExprResult { - match self { - Self::Leaf { value } => Ok(value.to_string().into()), - Self::BinOp { - op_type, - left, - right, - } => op_type.eval(left, right), - Self::Substr { - string, - pos, - length, - } => { - let string: String = string.eval()?.eval_as_string(); - - // The GNU docs say: - // - // > If either position or length is negative, zero, or - // > non-numeric, returns the null string. - // - // So we coerce errors into 0 to make that the only case we - // have to care about. - let pos = pos - .eval()? - .eval_as_bigint() - .ok() - .and_then(|n| n.to_usize()) - .unwrap_or(0); - let length = length - .eval()? - .eval_as_bigint() - .ok() - .and_then(|n| n.to_usize()) - .unwrap_or(0); - - let (Some(pos), Some(_)) = (pos.checked_sub(1), length.checked_sub(1)) else { - return Ok(String::new().into()); - }; + // This function implements a recursive tree-walking algorithm, but uses an explicit + // stack approach instead of native recursion to avoid potential stack overflow + // on deeply nested expressions. + + let mut stack = vec![self]; + let mut result_stack = BTreeMap::new(); + + while let Some(node) = stack.pop() { + match &node.inner { + AstNodeInner::Evaluated { value, .. } => { + result_stack.insert(node.id, Ok(value.clone())); + } + AstNodeInner::Leaf { value, .. } => { + result_stack.insert(node.id, Ok(value.to_string().into())); + } + AstNodeInner::BinOp { + op_type, + left, + right, + } => { + let (Some(right), Some(left)) = ( + result_stack.remove(&right.id), + result_stack.remove(&left.id), + ) else { + stack.push(node); + stack.push(right); + stack.push(left); + continue; + }; + + let result = op_type.eval(left, right); + result_stack.insert(node.id, result); + } + AstNodeInner::Substr { + string, + pos, + length, + } => { + let (Some(string), Some(pos), Some(length)) = ( + result_stack.remove(&string.id), + result_stack.remove(&pos.id), + result_stack.remove(&length.id), + ) else { + stack.push(node); + stack.push(string); + stack.push(pos); + stack.push(length); + continue; + }; + + let string: String = string?.eval_as_string(); + + // The GNU docs say: + // + // > If either position or length is negative, zero, or + // > non-numeric, returns the null string. + // + // So we coerce errors into 0 to make that the only case we + // have to care about. + let pos = pos? + .eval_as_bigint() + .ok() + .and_then(|n| n.to_usize()) + .unwrap_or(0); + let length = length? + .eval_as_bigint() + .ok() + .and_then(|n| n.to_usize()) + .unwrap_or(0); + + if let (Some(pos), Some(_)) = (pos.checked_sub(1), length.checked_sub(1)) { + let result = string.chars().skip(pos).take(length).collect::(); + result_stack.insert(node.id, Ok(result.into())); + } else { + result_stack.insert(node.id, Ok(String::new().into())); + } + } + AstNodeInner::Length { string } => { + // Push onto the stack - Ok(string - .chars() - .skip(pos) - .take(length) - .collect::() - .into()) + let Some(string) = result_stack.remove(&string.id) else { + stack.push(node); + stack.push(string); + continue; + }; + + let length = string?.eval_as_string().chars().count(); + result_stack.insert(node.id, Ok(length.into())); + } } - Self::Length { string } => Ok(string.eval()?.eval_as_string().chars().count().into()), } + + // The final result should be the only one left on the result stack + result_stack.remove(&self.id).unwrap() } } -struct Parser<'a> { - input: &'a [&'a str], +thread_local! { + static NODE_ID: Cell = const { Cell::new(1) }; +} + +// We create unique identifiers for each node in the AST. +// This is used to transform the recursive algorithm into an iterative one. +// It is used to store the result of each node's evaluation in a BtreeMap. +fn get_next_id() -> u32 { + NODE_ID.with(|id| { + let current = id.get(); + id.set(current + 1); + current + }) +} + +struct Parser<'a, S: AsRef> { + input: &'a [S], index: usize, } -impl<'a> Parser<'a> { - fn new(input: &'a [&'a str]) -> Self { +impl<'a, S: AsRef> Parser<'a, S> { + fn new(input: &'a [S]) -> Self { Self { input, index: 0 } } @@ -336,19 +553,19 @@ impl<'a> Parser<'a> { let next = self.input.get(self.index); if let Some(next) = next { self.index += 1; - Ok(next) + Ok(next.as_ref()) } else { // The indexing won't panic, because we know that the input size // is greater than zero. Err(ExprError::MissingArgument( - self.input[self.index - 1].into(), + self.input[self.index - 1].as_ref().into(), )) } } fn accept(&mut self, f: impl Fn(&str) -> Option) -> Option { let next = self.input.get(self.index)?; - let tok = f(next); + let tok = f(next.as_ref()); if let Some(tok) = tok { self.index += 1; Some(tok) @@ -363,7 +580,7 @@ impl<'a> Parser<'a> { } let res = self.parse_expression()?; if let Some(arg) = self.input.get(self.index) { - return Err(ExprError::UnexpectedArgument(arg.to_string())); + return Err(ExprError::UnexpectedArgument(arg.as_ref().into())); } Ok(res) } @@ -391,10 +608,13 @@ impl<'a> Parser<'a> { let mut left = self.parse_precedence(precedence + 1)?; while let Some(op) = self.parse_op(precedence) { let right = self.parse_precedence(precedence + 1)?; - left = AstNode::BinOp { - op_type: op, - left: Box::new(left), - right: Box::new(right), + left = AstNode { + id: get_next_id(), + inner: AstNodeInner::BinOp { + op_type: op, + left: Box::new(left), + right: Box::new(right), + }, }; } Ok(left) @@ -402,11 +622,11 @@ impl<'a> Parser<'a> { fn parse_simple_expression(&mut self) -> ExprResult { let first = self.next()?; - Ok(match first { + let inner = match first { "match" => { let left = self.parse_expression()?; let right = self.parse_expression()?; - AstNode::BinOp { + AstNodeInner::BinOp { op_type: BinOp::String(StringOp::Match), left: Box::new(left), right: Box::new(right), @@ -416,7 +636,7 @@ impl<'a> Parser<'a> { let string = self.parse_expression()?; let pos = self.parse_expression()?; let length = self.parse_expression()?; - AstNode::Substr { + AstNodeInner::Substr { string: Box::new(string), pos: Box::new(pos), length: Box::new(length), @@ -425,7 +645,7 @@ impl<'a> Parser<'a> { "index" => { let left = self.parse_expression()?; let right = self.parse_expression()?; - AstNode::BinOp { + AstNodeInner::BinOp { op_type: BinOp::String(StringOp::Index), left: Box::new(left), right: Box::new(right), @@ -433,26 +653,41 @@ impl<'a> Parser<'a> { } "length" => { let string = self.parse_expression()?; - AstNode::Length { + AstNodeInner::Length { string: Box::new(string), } } - "+" => AstNode::Leaf { + "+" => AstNodeInner::Leaf { value: self.next()?.into(), }, "(" => { - let s = self.parse_expression()?; - let close_paren = self.next()?; - if close_paren != ")" { + // Evaluate the node just after parsing to we detect arithmetic + // errors before checking for the closing parenthesis. + let s = self.parse_expression()?.evaluated()?; + + match self.next() { + Ok(")") => {} // Since we have parsed at least a '(', there will be a token // at `self.index - 1`. So this indexing won't panic. - return Err(ExprError::ExpectedClosingBraceAfter( - self.input[self.index - 1].into(), - )); + Ok(_) => { + return Err(ExprError::ExpectedClosingBraceInsteadOf( + self.input[self.index - 1].as_ref().into(), + )); + } + Err(ExprError::MissingArgument(_)) => { + return Err(ExprError::ExpectedClosingBraceAfter( + self.input[self.index - 1].as_ref().into(), + )); + } + Err(e) => return Err(e), } - s + s.inner } - s => AstNode::Leaf { value: s.into() }, + s => AstNodeInner::Leaf { value: s.into() }, + }; + Ok(AstNode { + id: get_next_id(), + inner, }) } } @@ -484,27 +719,50 @@ pub fn is_truthy(s: &NumOrStr) -> bool { #[cfg(test)] mod test { - use super::{AstNode, BinOp, NumericOp, RelationOp, StringOp}; + use crate::ExprError; + use crate::ExprError::InvalidBracketContent; + + use super::{ + AstNode, AstNodeInner, BinOp, NumericOp, RelationOp, StringOp, check_posix_regex_errors, + get_next_id, + }; + + impl PartialEq for AstNode { + fn eq(&self, other: &Self) -> bool { + self.inner == other.inner + } + } + + impl Eq for AstNode {} impl From<&str> for AstNode { fn from(value: &str) -> Self { - Self::Leaf { - value: value.into(), + Self { + id: get_next_id(), + inner: AstNodeInner::Leaf { + value: value.into(), + }, } } } fn op(op_type: BinOp, left: impl Into, right: impl Into) -> AstNode { - AstNode::BinOp { - op_type, - left: Box::new(left.into()), - right: Box::new(right.into()), + AstNode { + id: get_next_id(), + inner: AstNodeInner::BinOp { + op_type, + left: Box::new(left.into()), + right: Box::new(right.into()), + }, } } fn length(string: impl Into) -> AstNode { - AstNode::Length { - string: Box::new(string.into()), + AstNode { + id: get_next_id(), + inner: AstNodeInner::Length { + string: Box::new(string.into()), + }, } } @@ -513,10 +771,13 @@ mod test { pos: impl Into, length: impl Into, ) -> AstNode { - AstNode::Substr { - string: Box::new(string.into()), - pos: Box::new(pos.into()), - length: Box::new(length.into()), + AstNode { + id: get_next_id(), + inner: AstNodeInner::Substr { + string: Box::new(string.into()), + pos: Box::new(pos.into()), + length: Box::new(length.into()), + }, } } @@ -553,7 +814,7 @@ mod test { AstNode::parse(&["index", "1", "2"]), Ok(op(BinOp::String(StringOp::Index), "1", "2")), ); - assert_eq!(AstNode::parse(&["length", "1"]), Ok(length("1")),); + assert_eq!(AstNode::parse(&["length", "1"]), Ok(length("1"))); assert_eq!( AstNode::parse(&["substr", "1", "2", "3"]), Ok(substr("1", "2", "3")), @@ -574,7 +835,9 @@ mod test { AstNode::parse(&["(", "1", "+", "2", ")", "*", "3"]), Ok(op( BinOp::Numeric(NumericOp::Mul), - op(BinOp::Numeric(NumericOp::Add), "1", "2"), + op(BinOp::Numeric(NumericOp::Add), "1", "2") + .evaluated() + .unwrap(), "3" )) ); @@ -587,4 +850,123 @@ mod test { )), ); } + + #[test] + fn missing_closing_parenthesis() { + assert_eq!( + AstNode::parse(&["(", "42"]), + Err(ExprError::ExpectedClosingBraceAfter("42".to_string())) + ); + assert_eq!( + AstNode::parse(&["(", "42", "a"]), + Err(ExprError::ExpectedClosingBraceInsteadOf("a".to_string())) + ); + } + + #[test] + fn empty_substitution() { + // causes a panic in 0.0.25 + let result = AstNode::parse(&["a", ":", r"\(b\)*"]) + .unwrap() + .eval() + .unwrap(); + assert_eq!(result.eval_as_string(), ""); + } + + #[test] + fn starting_stars_become_escaped() { + let result = AstNode::parse(&["cats", ":", r"*cats"]) + .unwrap() + .eval() + .unwrap(); + assert_eq!(result.eval_as_string(), "0"); + + let result = AstNode::parse(&["*cats", ":", r"*cats"]) + .unwrap() + .eval() + .unwrap(); + assert_eq!(result.eval_as_string(), "5"); + } + + #[test] + fn only_match_in_beginning() { + let result = AstNode::parse(&["budget", ":", r"get"]) + .unwrap() + .eval() + .unwrap(); + assert_eq!(result.eval_as_string(), "0"); + } + + #[test] + fn check_regex_valid() { + assert!(check_posix_regex_errors(r"(a+b) \(a* b\)").is_ok()); + } + + #[test] + fn check_regex_simple_repeating_pattern() { + assert!(check_posix_regex_errors(r"\(a+b\)\{4\}").is_ok()); + } + + #[test] + fn check_regex_missing_closing() { + assert_eq!( + check_posix_regex_errors(r"\(abc"), + Err(ExprError::UnmatchedOpeningParenthesis) + ); + + assert_eq!( + check_posix_regex_errors(r"\{1,2"), + Err(ExprError::UnmatchedOpeningBrace) + ); + } + + #[test] + fn check_regex_missing_opening() { + assert_eq!( + check_posix_regex_errors(r"abc\)"), + Err(ExprError::UnmatchedClosingParenthesis) + ); + + assert_eq!( + check_posix_regex_errors(r"abc\}"), + Err(ExprError::UnmatchedClosingBrace) + ); + } + + #[test] + fn check_regex_empty_repeating_pattern() { + assert_eq!( + check_posix_regex_errors("ab\\{\\}"), + Err(InvalidBracketContent) + ); + } + + #[test] + fn check_regex_intervals_two_numbers() { + assert_eq!( + // out of order + check_posix_regex_errors("ab\\{1,0\\}"), + Err(InvalidBracketContent) + ); + assert_eq!( + check_posix_regex_errors("ab\\{1,a\\}"), + Err(InvalidBracketContent) + ); + assert_eq!( + check_posix_regex_errors("ab\\{a,3\\}"), + Err(InvalidBracketContent) + ); + assert_eq!( + check_posix_regex_errors("ab\\{a,b\\}"), + Err(InvalidBracketContent) + ); + assert_eq!( + check_posix_regex_errors("ab\\{a,\\}"), + Err(InvalidBracketContent) + ); + assert_eq!( + check_posix_regex_errors("ab\\{,b\\}"), + Err(InvalidBracketContent) + ); + } } diff --git a/src/uu/factor/Cargo.toml b/src/uu/factor/Cargo.toml index a149056888a..6b875074ec5 100644 --- a/src/uu/factor/Cargo.toml +++ b/src/uu/factor/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_factor" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "factor ~ (uutils) display the prime factors of each NUMBER" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [build-dependencies] num-traits = { workspace = true } # used in src/numerics.rs, which is included by build.rs @@ -21,13 +24,12 @@ num-traits = { workspace = true } rand = { workspace = true } smallvec = { workspace = true } uucore = { workspace = true } - -[dev-dependencies] -quickcheck = "1.0.3" +num-bigint = { workspace = true } +num-prime = { workspace = true } [[bin]] name = "factor" path = "src/main.rs" [lib] -path = "src/cli.rs" +path = "src/factor.rs" diff --git a/src/uu/factor/build.rs b/src/uu/factor/build.rs deleted file mode 100644 index 8de0605a292..00000000000 --- a/src/uu/factor/build.rs +++ /dev/null @@ -1,103 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -//! Generate a table of the multiplicative inverses of p_i mod 2^64 -//! for the first 1027 odd primes (all 13 bit and smaller primes). -//! You can supply a command line argument to override the default -//! value of 1027 for the number of entries in the table. -//! -//! 2 has no multiplicative inverse mode 2^64 because 2 | 2^64, -//! and in any case divisibility by two is trivial by checking the LSB. - -#![cfg_attr(test, allow(dead_code))] - -use std::env::{self, args}; -use std::fs::File; -use std::io::Write; -use std::path::Path; - -use self::sieve::Sieve; - -#[cfg(test)] -use miller_rabin::is_prime; - -#[path = "src/numeric/modular_inverse.rs"] -mod modular_inverse; -#[allow(unused_imports)] // imports there are used, but invisible from build.rs -#[path = "src/numeric/traits.rs"] -mod traits; -use modular_inverse::modular_inverse; - -mod sieve; - -#[cfg_attr(test, allow(dead_code))] -fn main() { - let out_dir = env::var("OUT_DIR").unwrap(); - let mut file = File::create(Path::new(&out_dir).join("prime_table.rs")).unwrap(); - - // By default, we print the multiplicative inverses mod 2^64 of the first 1k primes - const DEFAULT_SIZE: usize = 320; - let n = args() - .nth(1) - .and_then(|s| s.parse::().ok()) - .unwrap_or(DEFAULT_SIZE); - - write!(file, "{PREAMBLE}").unwrap(); - let mut cols = 3; - - // we want a total of n + 1 values - let mut primes = Sieve::odd_primes().take(n + 1); - - // in each iteration of the for loop, we use the value yielded - // by the previous iteration. This leaves one value left at the - // end, which we call NEXT_PRIME. - let mut x = primes.next().unwrap(); - for next in primes { - // format the table - let output = format!("({}, {}, {}),", x, modular_inverse(x), std::u64::MAX / x); - if cols + output.len() > MAX_WIDTH { - write!(file, "\n {output}").unwrap(); - cols = 4 + output.len(); - } else { - write!(file, " {output}").unwrap(); - cols += 1 + output.len(); - } - - x = next; - } - - write!( - file, - "\n];\n\n#[allow(dead_code)]\npub const NEXT_PRIME: u64 = {x};\n" - ) - .unwrap(); -} - -#[test] -fn test_generator_is_prime() { - assert_eq!(Sieve::odd_primes.take(10_000).all(is_prime)); -} - -#[test] -fn test_generator_10001() { - let prime_10001 = Sieve::primes().skip(10_000).next(); - assert_eq!(prime_10001, Some(104_743)); -} - -const MAX_WIDTH: usize = 102; -const PREAMBLE: &str = r##"/* -* This file is part of the uutils coreutils package. -* -* For the full copyright and license information, please view the LICENSE file -* that was distributed with this source code. -*/ - -// *** NOTE: this file was automatically generated. -// Please do not edit by hand. Instead, modify and -// re-run src/factor/gen_tables.rs. - -#[allow(clippy::unreadable_literal)] -pub const PRIME_INVERSIONS_U64: &[(u64, u64, u64)] = &[ - "##; diff --git a/src/uu/factor/sieve.rs b/src/uu/factor/sieve.rs deleted file mode 100644 index e2211ce051a..00000000000 --- a/src/uu/factor/sieve.rs +++ /dev/null @@ -1,215 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -// spell-checker:ignore (ToDO) filts, minidx, minkey paridx - -use std::iter::{Chain, Copied, Cycle}; -use std::slice::Iter; - -/// A lazy Sieve of Eratosthenes. -/// -/// This is a reasonably efficient implementation based on -/// O'Neill, M. E. "[The Genuine Sieve of Eratosthenes.](http://dx.doi.org/10.1017%2FS0956796808007004)" -/// Journal of Functional Programming, Volume 19, Issue 1, 2009, pp. 95--106. -#[derive(Default)] -pub struct Sieve { - inner: Wheel, - filts: PrimeHeap, -} - -impl Iterator for Sieve { - type Item = u64; - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } - - #[inline] - fn next(&mut self) -> Option { - for n in &mut self.inner { - let mut prime = true; - while let Some((next, inc)) = self.filts.peek() { - // need to keep checking the min element of the heap - // until we've found an element that's greater than n - if next > n { - break; // next heap element is bigger than n - } - - if next == n { - // n == next, and is composite. - prime = false; - } - // Increment the element in the prime heap. - self.filts.replace((next + inc, inc)); - } - - if prime { - // this is a prime; add it to the heap - self.filts.insert(n); - return Some(n); - } - } - None - } -} - -impl Sieve { - fn new() -> Self { - Self::default() - } - - #[allow(dead_code)] - #[inline] - pub fn primes() -> PrimeSieve { - INIT_PRIMES.iter().copied().chain(Self::new()) - } - - #[allow(dead_code)] - #[inline] - pub fn odd_primes() -> PrimeSieve { - INIT_PRIMES[1..].iter().copied().chain(Self::new()) - } -} - -pub type PrimeSieve = Chain>, Sieve>; - -/// An iterator that generates an infinite list of numbers that are -/// not divisible by any of 2, 3, 5, or 7. -struct Wheel { - next: u64, - increment: Cycle>, -} - -impl Iterator for Wheel { - type Item = u64; - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (1, None) - } - - #[inline] - fn next(&mut self) -> Option { - let increment = self.increment.next().unwrap(); // infinite iterator, no check necessary - let ret = self.next; - self.next = ret + increment; - Some(ret) - } -} - -impl Wheel { - #[inline] - fn new() -> Self { - Self { - next: 11u64, - increment: WHEEL_INCS.iter().cycle(), - } - } -} - -impl Default for Wheel { - fn default() -> Self { - Self::new() - } -} - -/// The increments of a wheel of circumference 210 -/// (i.e., a wheel that skips all multiples of 2, 3, 5, 7) -const WHEEL_INCS: &[u64] = &[ - 2, 4, 2, 4, 6, 2, 6, 4, 2, 4, 6, 6, 2, 6, 4, 2, 6, 4, 6, 8, 4, 2, 4, 2, 4, 8, 6, 4, 6, 2, 4, 6, - 2, 6, 6, 4, 2, 4, 6, 2, 6, 4, 2, 4, 2, 10, 2, 10, -]; -const INIT_PRIMES: &[u64] = &[2, 3, 5, 7]; - -/// A min-heap of "infinite lists" of prime multiples, where a list is -/// represented as (head, increment). -#[derive(Debug, Default)] -struct PrimeHeap { - data: Vec<(u64, u64)>, -} - -impl PrimeHeap { - fn peek(&self) -> Option<(u64, u64)> { - if let Some(&(x, y)) = self.data.first() { - Some((x, y)) - } else { - None - } - } - - fn insert(&mut self, next: u64) { - let mut idx = self.data.len(); - let key = next * next; - - let item = (key, next); - self.data.push(item); - loop { - // break if we've bubbled to the top - if idx == 0 { - break; - } - - let paridx = (idx - 1) / 2; - let (k, _) = self.data[paridx]; - if key < k { - // bubble up, found a smaller key - self.data.swap(idx, paridx); - idx = paridx; - } else { - // otherwise, parent is smaller, so we're done - break; - } - } - } - - fn remove(&mut self) -> (u64, u64) { - let ret = self.data.swap_remove(0); - - let mut idx = 0; - let len = self.data.len(); - let (key, _) = self.data[0]; - loop { - let child1 = 2 * idx + 1; - let child2 = 2 * idx + 2; - - // no more children - if child1 >= len { - break; - } - - // find lesser child - let (c1key, _) = self.data[child1]; - let (minidx, minkey) = if child2 >= len { - (child1, c1key) - } else { - let (c2key, _) = self.data[child2]; - if c1key < c2key { - (child1, c1key) - } else { - (child2, c2key) - } - }; - - if minkey < key { - self.data.swap(minidx, idx); - idx = minidx; - continue; - } - - // smaller than both children, so done - break; - } - - ret - } - - /// More efficient than inserting and removing in two steps - /// because we save one traversal of the heap. - fn replace(&mut self, next: (u64, u64)) -> (u64, u64) { - self.data.push(next); - self.remove() - } -} diff --git a/src/uu/factor/src/cli.rs b/src/uu/factor/src/cli.rs deleted file mode 100644 index d01ca625c7b..00000000000 --- a/src/uu/factor/src/cli.rs +++ /dev/null @@ -1,120 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -use std::io::BufRead; -use std::io::{self, stdin, stdout, Write}; - -mod factor; -use clap::{crate_version, Arg, ArgAction, Command}; -pub use factor::*; -use uucore::display::Quotable; -use uucore::error::{set_exit_code, FromIo, UResult}; -use uucore::{format_usage, help_about, help_usage, show_error, show_warning}; - -mod miller_rabin; -pub mod numeric; -mod rho; -pub mod table; - -const ABOUT: &str = help_about!("factor.md"); -const USAGE: &str = help_usage!("factor.md"); - -mod options { - pub static EXPONENTS: &str = "exponents"; - pub static HELP: &str = "help"; - pub static NUMBER: &str = "NUMBER"; -} - -fn print_factors_str( - num_str: &str, - w: &mut io::BufWriter, - print_exponents: bool, -) -> io::Result<()> { - let x = match num_str.trim().parse::() { - Ok(x) => x, - Err(e) => { - // We return Ok() instead of Err(), because it's non-fatal and we should try the next - // number. - show_warning!("{}: {}", num_str.maybe_quote(), e); - set_exit_code(1); - return Ok(()); - } - }; - - // If print_exponents is true, use the alternate format specifier {:#} from fmt to print the factors - // of x in the form of p^e. - if print_exponents { - writeln!(w, "{}:{:#}", x, factor(x))?; - } else { - writeln!(w, "{}:{}", x, factor(x))?; - } - w.flush() -} - -#[uucore::main] -pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let matches = uu_app().try_get_matches_from(args)?; - - // If matches find --exponents flag than variable print_exponents is true and p^e output format will be used. - let print_exponents = matches.get_flag(options::EXPONENTS); - - let stdout = stdout(); - // We use a smaller buffer here to pass a gnu test. 4KiB appears to be the default pipe size for bash. - let mut w = io::BufWriter::with_capacity(4 * 1024, stdout.lock()); - - if let Some(values) = matches.get_many::(options::NUMBER) { - for number in values { - print_factors_str(number, &mut w, print_exponents) - .map_err_context(|| "write error".into())?; - } - } else { - let stdin = stdin(); - let lines = stdin.lock().lines(); - for line in lines { - match line { - Ok(line) => { - for number in line.split_whitespace() { - print_factors_str(number, &mut w, print_exponents) - .map_err_context(|| "write error".into())?; - } - } - Err(e) => { - set_exit_code(1); - show_error!("error reading input: {}", e); - return Ok(()); - } - } - } - } - - if let Err(e) = w.flush() { - show_error!("{}", e); - } - - Ok(()) -} - -pub fn uu_app() -> Command { - Command::new(uucore::util_name()) - .version(crate_version!()) - .about(ABOUT) - .override_usage(format_usage(USAGE)) - .infer_long_args(true) - .disable_help_flag(true) - .arg(Arg::new(options::NUMBER).action(ArgAction::Append)) - .arg( - Arg::new(options::EXPONENTS) - .short('h') - .long(options::EXPONENTS) - .help("Print factors in the form p^e") - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::HELP) - .long(options::HELP) - .help("Print help information.") - .action(ArgAction::Help), - ) -} diff --git a/src/uu/factor/src/factor.rs b/src/uu/factor/src/factor.rs index d7899a7e6ee..2c8d1661c05 100644 --- a/src/uu/factor/src/factor.rs +++ b/src/uu/factor/src/factor.rs @@ -3,302 +3,142 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -#![allow(clippy::items_after_test_module)] -use smallvec::SmallVec; -use std::cell::RefCell; -use std::fmt; - -use crate::numeric::{Arithmetic, Montgomery}; -use crate::{miller_rabin, rho, table}; - -type Exponent = u8; - -#[derive(Clone, Debug, Default)] -struct Decomposition(SmallVec<[(u64, Exponent); NUM_FACTORS_INLINE]>); - -// spell-checker:ignore (names) ErdÅ‘s–Kac * ErdÅ‘s Kac -// The number of factors to inline directly into a `Decomposition` object. -// As a consequence of the ErdÅ‘s–Kac theorem, the average number of prime factors -// of integers < 10²ⵠ≃ 2â¸Â³ is 4, so we can use a slightly higher value. -const NUM_FACTORS_INLINE: usize = 5; - -impl Decomposition { - fn one() -> Self { - Self::default() - } - - fn add(&mut self, factor: u64, exp: Exponent) { - debug_assert!(exp > 0); - - if let Some((_, e)) = self.0.iter_mut().find(|(f, _)| *f == factor) { - *e += exp; - } else { - self.0.push((factor, exp)); - } - } - - #[cfg(test)] - fn product(&self) -> u64 { - self.0 - .iter() - .fold(1, |acc, (p, exp)| acc * p.pow(*exp as u32)) - } - - fn get(&self, p: u64) -> Option<&(u64, u8)> { - self.0.iter().find(|(q, _)| *q == p) - } -} - -impl PartialEq for Decomposition { - fn eq(&self, other: &Self) -> bool { - for p in &self.0 { - if other.get(p.0) != Some(p) { - return false; - } - } - - for p in &other.0 { - if self.get(p.0) != Some(p) { - return false; - } - } - - true - } +// spell-checker:ignore funcs + +use std::collections::BTreeMap; +use std::io::BufRead; +use std::io::{self, Write, stdin, stdout}; + +use clap::{Arg, ArgAction, Command}; +use num_bigint::BigUint; +use num_traits::FromPrimitive; +use uucore::display::Quotable; +use uucore::error::{FromIo, UResult, USimpleError, set_exit_code}; +use uucore::{format_usage, help_about, help_usage, show_error, show_warning}; + +const ABOUT: &str = help_about!("factor.md"); +const USAGE: &str = help_usage!("factor.md"); + +mod options { + pub static EXPONENTS: &str = "exponents"; + pub static HELP: &str = "help"; + pub static NUMBER: &str = "NUMBER"; } -impl Eq for Decomposition {} -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct Factors(RefCell); +fn print_factors_str( + num_str: &str, + w: &mut io::BufWriter, + print_exponents: bool, +) -> UResult<()> { + let rx = num_str.trim().parse::(); + let Ok(x) = rx else { + // return Ok(). it's non-fatal and we should try the next number. + show_warning!("{}: {}", num_str.maybe_quote(), rx.unwrap_err()); + set_exit_code(1); + return Ok(()); + }; -impl Factors { - pub fn one() -> Self { - Self(RefCell::new(Decomposition::one())) - } + let (factorization, remaining) = if x > BigUint::from_u32(1).unwrap() { + num_prime::nt_funcs::factors(x.clone(), None) + } else { + (BTreeMap::new(), None) + }; - pub fn add(&mut self, prime: u64, exp: Exponent) { - debug_assert!(miller_rabin::is_prime(prime)); - self.0.borrow_mut().add(prime, exp); + if let Some(_remaining) = remaining { + return Err(USimpleError::new( + 1, + "Factorization incomplete. Remainders exists.", + )); } - pub fn push(&mut self, prime: u64) { - self.add(prime, 1); - } + write_result(w, &x, factorization, print_exponents).map_err_context(|| "write error".into())?; - #[cfg(test)] - fn product(&self) -> u64 { - self.0.borrow().product() - } + Ok(()) } -impl fmt::Display for Factors { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let v = &mut (self.0).borrow_mut().0; - v.sort_unstable(); - - let include_exponents = f.alternate(); - for (p, exp) in v { - if include_exponents && *exp > 1 { - write!(f, " {p}^{exp}")?; +fn write_result( + w: &mut io::BufWriter, + x: &BigUint, + factorization: BTreeMap, + print_exponents: bool, +) -> io::Result<()> { + write!(w, "{x}:")?; + for (factor, n) in factorization { + if print_exponents { + if n > 1 { + write!(w, " {factor}^{n}")?; } else { - for _ in 0..*exp { - write!(f, " {p}")?; - } + write!(w, " {factor}")?; } - } - - Ok(()) - } -} - -fn _factor(num: u64, f: Factors) -> Factors { - use miller_rabin::Result::*; - - // Shadow the name, so the recursion automatically goes from “Big†arithmetic to small. - let _factor = |n, f| { - if n < (1 << 32) { - _factor::>(n, f) } else { - _factor::(n, f) - } - }; - - if num == 1 { - return f; - } - - let n = A::new(num); - let divisor = match miller_rabin::test::(n) { - Prime => { - #[cfg(feature = "coz")] - coz::progress!("factor found"); - let mut r = f; - r.push(num); - return r; + w.write_all(format!(" {factor}").repeat(n).as_bytes())?; } - - Composite(d) => d, - Pseudoprime => rho::find_divisor::(n), - }; - - let f = _factor(divisor, f); - _factor(num / divisor, f) -} - -pub fn factor(mut n: u64) -> Factors { - #[cfg(feature = "coz")] - coz::begin!("factorization"); - let mut factors = Factors::one(); - - if n < 2 { - return factors; - } - - let n_zeros = n.trailing_zeros(); - if n_zeros > 0 { - factors.add(2, n_zeros as Exponent); - n >>= n_zeros; - } - - if n == 1 { - #[cfg(feature = "coz")] - coz::end!("factorization"); - return factors; } - - table::factor(&mut n, &mut factors); - - #[allow(clippy::let_and_return)] - let r = if n < (1 << 32) { - _factor::>(n, factors) - } else { - _factor::>(n, factors) - }; - - #[cfg(feature = "coz")] - coz::end!("factorization"); - - r + writeln!(w)?; + w.flush() } -#[cfg(test)] -mod tests { - use super::{factor, Decomposition, Exponent, Factors}; - use quickcheck::quickcheck; - use smallvec::smallvec; - use std::cell::RefCell; - - #[test] - fn factor_2044854919485649() { - let f = Factors(RefCell::new(Decomposition(smallvec![ - (503, 1), - (2423, 1), - (40961, 2) - ]))); - assert_eq!(factor(f.product()), f); - } - - #[test] - fn factor_recombines_small() { - assert!((1..10_000) - .map(|i| 2 * i + 1) - .all(|i| factor(i).product() == i)); - } - - #[test] - fn factor_recombines_overflowing() { - assert!((0..250) - .map(|i| 2 * i + 2u64.pow(32) + 1) - .all(|i| factor(i).product() == i)); - } +#[uucore::main] +pub fn uumain(args: impl uucore::Args) -> UResult<()> { + let matches = uu_app().try_get_matches_from(args)?; - #[test] - fn factor_recombines_strong_pseudoprime() { - // This is a strong pseudoprime (wrt. miller_rabin::BASIS) - // and triggered a bug in rho::factor's code path handling - // miller_rabbin::Result::Composite - let pseudoprime = 17179869183; - for _ in 0..20 { - // Repeat the test 20 times, as it only fails some fraction - // of the time. - assert!(factor(pseudoprime).product() == pseudoprime); - } - } + // If matches find --exponents flag than variable print_exponents is true and p^e output format will be used. + let print_exponents = matches.get_flag(options::EXPONENTS); - quickcheck! { - fn factor_recombines(i: u64) -> bool { - i == 0 || factor(i).product() == i - } + let stdout = stdout(); + // We use a smaller buffer here to pass a gnu test. 4KiB appears to be the default pipe size for bash. + let mut w = io::BufWriter::with_capacity(4 * 1024, stdout.lock()); - fn recombines_factors(f: Factors) -> () { - assert_eq!(factor(f.product()), f); + if let Some(values) = matches.get_many::(options::NUMBER) { + for number in values { + print_factors_str(number, &mut w, print_exponents)?; } - - fn exponentiate_factors(f: Factors, e: Exponent) -> () { - if e == 0 { return; } - if let Some(fe) = f.product().checked_pow(e.into()) { - assert_eq!(factor(fe), f ^ e); - } - } - } -} - -#[cfg(test)] -use rand::{ - distributions::{Distribution, Standard}, - Rng, -}; -#[cfg(test)] -impl Distribution for Standard { - fn sample(&self, rng: &mut R) -> Factors { - let mut f = Factors::one(); - let mut g = 1u64; - let mut n = u64::MAX; - - // spell-checker:ignore (names) Adam Kalai * Kalai's - // Adam Kalai's algorithm for generating uniformly-distributed - // integers and their factorization. - // - // See Generating Random Factored Numbers, Easily, J. Cryptology (2003) - 'attempt: loop { - while n > 1 { - n = rng.gen_range(1..n); - if miller_rabin::is_prime(n) { - if let Some(h) = g.checked_mul(n) { - f.push(n); - g = h; - } else { - // We are overflowing u64, retry - continue 'attempt; + } else { + let stdin = stdin(); + let lines = stdin.lock().lines(); + for line in lines { + match line { + Ok(line) => { + for number in line.split_whitespace() { + print_factors_str(number, &mut w, print_exponents)?; } } + Err(e) => { + set_exit_code(1); + show_error!("error reading input: {e}"); + return Ok(()); + } } - - return f; } } -} -#[cfg(test)] -impl quickcheck::Arbitrary for Factors { - fn arbitrary(g: &mut quickcheck::Gen) -> Self { - factor(u64::arbitrary(g)) + if let Err(e) = w.flush() { + show_error!("{e}"); } -} - -#[cfg(test)] -impl std::ops::BitXor for Factors { - type Output = Self; - fn bitxor(self, rhs: Exponent) -> Self { - debug_assert_ne!(rhs, 0); - let mut r = Self::one(); - #[allow(clippy::explicit_iter_loop)] - for (p, e) in self.0.borrow().0.iter() { - r.add(*p, rhs * e); - } + Ok(()) +} - debug_assert_eq!(r.product(), self.product().pow(rhs.into())); - r - } +pub fn uu_app() -> Command { + Command::new(uucore::util_name()) + .version(uucore::crate_version!()) + .about(ABOUT) + .override_usage(format_usage(USAGE)) + .infer_long_args(true) + .disable_help_flag(true) + .args_override_self(true) + .arg(Arg::new(options::NUMBER).action(ArgAction::Append)) + .arg( + Arg::new(options::EXPONENTS) + .short('h') + .long(options::EXPONENTS) + .help("Print factors in the form p^e") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::HELP) + .long(options::HELP) + .help("Print help information.") + .action(ArgAction::Help), + ) } diff --git a/src/uu/factor/src/miller_rabin.rs b/src/uu/factor/src/miller_rabin.rs deleted file mode 100644 index a06b20cd3b0..00000000000 --- a/src/uu/factor/src/miller_rabin.rs +++ /dev/null @@ -1,224 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -// spell-checker:ignore (URL) appspot - -use crate::numeric::*; - -pub(crate) trait Basis { - const BASIS: &'static [u64]; -} - -impl Basis for Montgomery { - // Small set of bases for the Miller-Rabin prime test, valid for all 64b integers; - // discovered by Jim Sinclair on 2011-04-20, see miller-rabin.appspot.com - #[allow(clippy::unreadable_literal)] - const BASIS: &'static [u64] = &[2, 325, 9375, 28178, 450775, 9780504, 1795265022]; -} - -impl Basis for Montgomery { - // spell-checker:ignore (names) Steve Worley - // Small set of bases for the Miller-Rabin prime test, valid for all 32b integers; - // discovered by Steve Worley on 2013-05-27, see miller-rabin.appspot.com - #[allow(clippy::unreadable_literal)] - const BASIS: &'static [u64] = &[ - 4230279247111683200, - 14694767155120705706, - 16641139526367750375, - ]; -} - -#[derive(Eq, PartialEq)] -#[must_use = "Ignoring the output of a primality test."] -pub(crate) enum Result { - Prime, - Pseudoprime, - Composite(u64), -} - -impl Result { - pub(crate) fn is_prime(&self) -> bool { - *self == Self::Prime - } -} - -// Deterministic Miller-Rabin primality-checking algorithm, adapted to extract -// (some) dividers; it will fail to factor strong pseudoprimes. -#[allow(clippy::many_single_char_names)] -pub(crate) fn test(m: A) -> Result { - use self::Result::*; - - let n = m.modulus(); - debug_assert!(n > 1); - debug_assert!(n % 2 != 0); - - // n-1 = r 2â± - let i = (n - 1).trailing_zeros(); - let r = (n - 1) >> i; - - let one = m.one(); - let minus_one = m.minus_one(); - - 'witness: for _a in A::BASIS { - let _a = _a % n; - if _a == 0 { - continue; - } - - let a = m.to_mod(_a); - - // x = a^r mod n - let mut x = m.pow(a, r); - - if x == one || x == minus_one { - continue; - } - - for _ in 1..i { - let y = m.mul(x, x); - if y == one { - return Composite(gcd(m.to_u64(x) - 1, m.modulus())); - } else if y == minus_one { - // This basis element is not a witness of `n` being composite. - // Keep looking. - continue 'witness; - } - x = y; - } - - return Pseudoprime; - } - - Prime -} - -// Used by build.rs' tests and debug assertions -#[allow(dead_code)] -pub(crate) fn is_prime(n: u64) -> bool { - if n < 2 { - false - } else if n % 2 == 0 { - n == 2 - } else { - test::>(Montgomery::new(n)).is_prime() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::numeric::{traits::DoubleInt, Arithmetic, Montgomery}; - use quickcheck::quickcheck; - use std::iter; - const LARGEST_U64_PRIME: u64 = 0xFFFFFFFFFFFFFFC5; - - fn primes() -> impl Iterator { - iter::once(2).chain(odd_primes()) - } - - fn odd_primes() -> impl Iterator { - use crate::table::{NEXT_PRIME, PRIME_INVERSIONS_U64}; - PRIME_INVERSIONS_U64 - .iter() - .map(|(p, _, _)| *p) - .chain(iter::once(NEXT_PRIME)) - } - - #[test] - fn largest_prime() { - assert!(is_prime(LARGEST_U64_PRIME)); - } - - #[test] - fn largest_composites() { - for i in LARGEST_U64_PRIME + 1..=u64::MAX { - assert!(!is_prime(i), "2â¶â´ - {} reported prime", u64::MAX - i + 1); - } - } - - #[test] - fn two() { - assert!(is_prime(2)); - } - - fn first_primes() - where - Montgomery: Basis, - { - for p in odd_primes() { - assert!( - test(Montgomery::::new(p)).is_prime(), - "{p} reported composite" - ); - } - } - - #[test] - fn first_primes_u32() { - first_primes::(); - } - - #[test] - fn first_primes_u64() { - first_primes::(); - } - - #[test] - fn one() { - assert!(!is_prime(1)); - } - #[test] - fn zero() { - assert!(!is_prime(0)); - } - - #[test] - fn first_composites() { - for (p, q) in primes().zip(odd_primes()) { - for i in p + 1..q { - assert!(!is_prime(i), "{i} reported prime"); - } - } - } - - #[test] - fn issue_1556() { - // 10 425 511 = 2441 × 4271 - assert!(!is_prime(10_425_511)); - } - - fn small_semiprimes() - where - Montgomery: Basis, - { - for p in odd_primes() { - for q in odd_primes().take_while(|q| *q <= p) { - let n = p * q; - let m = Montgomery::::new(n); - assert!(!test(m).is_prime(), "{n} = {p} × {q} reported prime"); - } - } - } - - #[test] - fn small_semiprimes_u32() { - small_semiprimes::(); - } - - #[test] - fn small_semiprimes_u64() { - small_semiprimes::(); - } - - quickcheck! { - fn composites(i: u64, j: u64) -> bool { - // TODO: #1559 factor n > 2^64 - 1 - match i.checked_mul(j) { - Some(n) => i < 2 || j < 2 || !is_prime(n), - _ => true, - } - } - } -} diff --git a/src/uu/factor/src/numeric/gcd.rs b/src/uu/factor/src/numeric/gcd.rs deleted file mode 100644 index 43c6ce9b71b..00000000000 --- a/src/uu/factor/src/numeric/gcd.rs +++ /dev/null @@ -1,126 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -// spell-checker:ignore (vars) kgcdab gcdac gcdbc - -use std::cmp::min; -use std::mem::swap; - -pub fn gcd(mut u: u64, mut v: u64) -> u64 { - // Stein's binary GCD algorithm - // Base cases: gcd(n, 0) = gcd(0, n) = n - if u == 0 { - return v; - } else if v == 0 { - return u; - } - - // gcd(2â± u, 2ʲ v) = 2áµ gcd(u, v) with u, v odd and k = min(i, j) - // 2áµ is the greatest power of two that divides both u and v - let k = { - let i = u.trailing_zeros(); - let j = v.trailing_zeros(); - u >>= i; - v >>= j; - min(i, j) - }; - - loop { - // Loop invariant: u and v are odd - debug_assert!(u % 2 == 1, "u = {u} is even"); - debug_assert!(v % 2 == 1, "v = {v} is even"); - - // gcd(u, v) = gcd(|u - v|, min(u, v)) - if u > v { - swap(&mut u, &mut v); - } - v -= u; - - if v == 0 { - // Reached the base case; gcd is 2áµ u - return u << k; - } - - // gcd(u, 2ʲ v) = gcd(u, v) as u is odd - v >>= v.trailing_zeros(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use quickcheck::{quickcheck, TestResult}; - - quickcheck! { - fn euclidean(a: u64, b: u64) -> bool { - // Test against the Euclidean algorithm - let g = { - let (mut a, mut b) = (a, b); - while b > 0 { - a %= b; - swap(&mut a, &mut b); - } - a - }; - gcd(a, b) == g - } - - fn one(a: u64) -> bool { - gcd(1, a) == 1 - } - - fn zero(a: u64) -> bool { - gcd(0, a) == a - } - - fn divisor(a: u64, b: u64) -> TestResult { - // Test that gcd(a, b) divides a and b, unless a == b == 0 - if a == 0 && b == 0 { return TestResult::discard(); } // restrict test domain to !(a == b == 0) - - let g = gcd(a, b); - TestResult::from_bool( g != 0 && a % g == 0 && b % g == 0 ) - } - - fn commutative(a: u64, b: u64) -> bool { - gcd(a, b) == gcd(b, a) - } - - fn associative(a: u64, b: u64, c: u64) -> bool { - gcd(a, gcd(b, c)) == gcd(gcd(a, b), c) - } - - fn scalar_multiplication(a: u64, b: u64, k: u64) -> bool { - // TODO: #1559 factor n > 2^64 - 1 - match (k.checked_mul(a), k.checked_mul(b), k.checked_mul(gcd(a, b))) { - (Some(ka), Some(kb), Some(kgcdab)) => gcd(ka, kb) == kgcdab, - _ => true - } - } - - fn multiplicative(a: u64, b: u64, c: u64) -> bool { - // TODO: #1559 factor n > 2^64 - 1 - match (a.checked_mul(b), gcd(a, c).checked_mul(gcd(b, c))) { - (Some(ab), Some(gcdac_gcdbc)) => { - // gcd(ab, c) = gcd(a, c) gcd(b, c) when a and b coprime - gcd(a, b) != 1 || gcd(ab, c) == gcdac_gcdbc - }, - _ => true, - } - } - - fn linearity(a: u64, b: u64, k: u64) -> bool { - // TODO: #1559 factor n > 2^64 - 1 - match k.checked_mul(b) { - Some(kb) => { - match a.checked_add(kb) { - Some(a_plus_kb) => gcd(a_plus_kb, b) == gcd(a, b), - _ => true, - } - } - _ => true, - } - } - } -} diff --git a/src/uu/factor/src/numeric/mod.rs b/src/uu/factor/src/numeric/mod.rs deleted file mode 100644 index d4c0b5dc7f7..00000000000 --- a/src/uu/factor/src/numeric/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -mod gcd; -pub use gcd::gcd; - -pub(crate) mod traits; - -mod modular_inverse; -pub(crate) use modular_inverse::modular_inverse; - -mod montgomery; -pub(crate) use montgomery::{Arithmetic, Montgomery}; diff --git a/src/uu/factor/src/numeric/modular_inverse.rs b/src/uu/factor/src/numeric/modular_inverse.rs deleted file mode 100644 index bacf57d9ce6..00000000000 --- a/src/uu/factor/src/numeric/modular_inverse.rs +++ /dev/null @@ -1,84 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -use super::traits::Int; - -// extended Euclid algorithm -// precondition: a is odd -pub(crate) fn modular_inverse(a: T) -> T { - let zero = T::zero(); - let one = T::one(); - debug_assert!(a % (one + one) == one, "{a:?} is not odd"); - - let mut t = zero; - let mut new_t = one; - let mut r = zero; - let mut new_r = a; - - while new_r != zero { - let quot = if r == zero { - // special case when we're just starting out - // This works because we know that - // a does not divide 2^64, so floor(2^64 / a) == floor((2^64-1) / a); - T::max_value() - } else { - r - } / new_r; - - let new_tp = t.wrapping_sub(".wrapping_mul(&new_t)); - t = new_t; - new_t = new_tp; - - let new_rp = r.wrapping_sub(".wrapping_mul(&new_r)); - r = new_r; - new_r = new_rp; - } - - debug_assert_eq!(r, one); - t -} - -#[cfg(test)] -mod tests { - use super::{super::traits::Int, *}; - use quickcheck::quickcheck; - - fn small_values() { - // All odd integers from 1 to 20 000 - let one = T::from(1).unwrap(); - let two = T::from(2).unwrap(); - let mut test_values = (0..10_000) - .map(|i| T::from(i).unwrap()) - .map(|i| two * i + one); - - assert!(test_values.all(|x| x.wrapping_mul(&modular_inverse(x)) == one)); - } - - #[test] - fn small_values_u32() { - small_values::(); - } - - #[test] - fn small_values_u64() { - small_values::(); - } - - quickcheck! { - fn random_values_u32(n: u32) -> bool { - match 2_u32.checked_mul(n) { - Some(n) => modular_inverse(n + 1).wrapping_mul(n + 1) == 1, - _ => true, - } - } - - fn random_values_u64(n: u64) -> bool { - match 2_u64.checked_mul(n) { - Some(n) => modular_inverse(n + 1).wrapping_mul(n + 1) == 1, - _ => true, - } - } - } -} diff --git a/src/uu/factor/src/numeric/montgomery.rs b/src/uu/factor/src/numeric/montgomery.rs deleted file mode 100644 index 10c6dd2d906..00000000000 --- a/src/uu/factor/src/numeric/montgomery.rs +++ /dev/null @@ -1,251 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -use super::*; - -use traits::{DoubleInt, Int, One, OverflowingAdd, Zero}; - -pub(crate) trait Arithmetic: Copy + Sized { - // The type of integers mod m, in some opaque representation - type ModInt: Copy + Sized + Eq; - - fn new(m: u64) -> Self; - fn modulus(&self) -> u64; - fn to_mod(&self, n: u64) -> Self::ModInt; - fn to_u64(&self, n: Self::ModInt) -> u64; - fn add(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt; - fn mul(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt; - - fn pow(&self, mut a: Self::ModInt, mut b: u64) -> Self::ModInt { - let (_a, _b) = (a, b); - let mut result = self.one(); - while b > 0 { - if b & 1 != 0 { - result = self.mul(result, a); - } - a = self.mul(a, a); - b >>= 1; - } - - // Check that r (reduced back to the usual representation) equals - // a^b % n, unless the latter computation overflows - // Temporarily commented-out, as there u64::checked_pow is not available - // on the minimum supported Rust version, nor is an appropriate method - // for compiling the check conditionally. - //debug_assert!(self - // .to_u64(_a) - // .checked_pow(_b as u32) - // .map(|r| r % self.modulus() == self.to_u64(result)) - // .unwrap_or(true)); - - result - } - - fn one(&self) -> Self::ModInt { - self.to_mod(1) - } - fn minus_one(&self) -> Self::ModInt { - self.to_mod(self.modulus() - 1) - } - fn zero(&self) -> Self::ModInt { - self.to_mod(0) - } -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct Montgomery { - a: T, - n: T, -} - -impl Montgomery { - /// computes x/R mod n efficiently - fn reduce(&self, x: T::DoubleWidth) -> T { - let t_bits = T::zero().count_zeros() as usize; - - debug_assert!(x < (self.n.as_double_width()) << t_bits); - // TODO: optimize - let Self { a, n } = self; - let m = T::from_double_width(x).wrapping_mul(a); - let nm = (n.as_double_width()) * (m.as_double_width()); - let (xnm, overflow) = x.overflowing_add(&nm); // x + n*m - debug_assert_eq!( - xnm % (T::DoubleWidth::one() << T::zero().count_zeros() as usize), - T::DoubleWidth::zero() - ); - - // (x + n*m) / R - // in case of overflow, this is (2¹²⸠+ xnm)/2â¶â´ - n = xnm/2â¶â´ + (2â¶â´ - n) - let y = T::from_double_width(xnm >> t_bits) - + if overflow { - n.wrapping_neg() - } else { - T::zero() - }; - - if y >= *n { - y - *n - } else { - y - } - } -} - -impl Arithmetic for Montgomery { - // Montgomery transform, R=2â¶â´ - // Provides fast arithmetic mod n (n odd, u64) - type ModInt = T; - - fn new(n: u64) -> Self { - debug_assert!(T::zero().count_zeros() >= 64 || n < (1 << T::zero().count_zeros() as usize)); - let n = T::from_u64(n); - let a = modular_inverse(n).wrapping_neg(); - debug_assert_eq!(n.wrapping_mul(&a), T::one().wrapping_neg()); - Self { a, n } - } - - fn modulus(&self) -> u64 { - self.n.as_u64() - } - - fn to_mod(&self, x: u64) -> Self::ModInt { - // TODO: optimise! - debug_assert!(x < self.n.as_u64()); - let r = T::from_double_width( - ((T::DoubleWidth::from_u64(x)) << T::zero().count_zeros() as usize) - % self.n.as_double_width(), - ); - debug_assert_eq!(x, self.to_u64(r)); - r - } - - fn to_u64(&self, n: Self::ModInt) -> u64 { - self.reduce(n.as_double_width()).as_u64() - } - - fn add(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt { - let (r, overflow) = a.overflowing_add(&b); - - // In case of overflow, a+b = 2â¶â´ + r = (2â¶â´ - n) + r (working mod n) - let r = if overflow { - r + self.n.wrapping_neg() - } else { - r - }; - - // Normalize to [0; n[ - let r = if r < self.n { r } else { r - self.n }; - - // Check that r (reduced back to the usual representation) equals - // a+b % n - #[cfg(debug_assertions)] - { - let a_r = self.to_u64(a) as u128; - let b_r = self.to_u64(b) as u128; - let r_r = self.to_u64(r); - let r_2 = ((a_r + b_r) % self.n.as_u128()) as u64; - debug_assert_eq!( - r_r, r_2, - "[{}] = {} ≠ {} = {} + {} = [{}] + [{}] mod {}; a = {}", - r, r_r, r_2, a_r, b_r, a, b, self.n, self.a - ); - } - r - } - - fn mul(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt { - let r = self.reduce(a.as_double_width() * b.as_double_width()); - - // Check that r (reduced back to the usual representation) equals - // a*b % n - #[cfg(debug_assertions)] - { - let a_r = self.to_u64(a) as u128; - let b_r = self.to_u64(b) as u128; - let r_r = self.to_u64(r); - let r_2: u64 = ((a_r * b_r) % self.n.as_u128()) as u64; - debug_assert_eq!( - r_r, r_2, - "[{}] = {} ≠ {} = {} * {} = [{}] * [{}] mod {}; a = {}", - r, r_r, r_2, a_r, b_r, a, b, self.n, self.a - ); - } - r - } -} - -#[cfg(test)] -mod tests { - use super::*; - fn test_add() { - for n in 0..100 { - let n = 2 * n + 1; - let m = Montgomery::::new(n); - for x in 0..n { - let m_x = m.to_mod(x); - for y in 0..=x { - let m_y = m.to_mod(y); - println!("{n:?}, {x:?}, {y:?}"); - assert_eq!((x + y) % n, m.to_u64(m.add(m_x, m_y))); - } - } - } - } - - #[test] - fn add_u32() { - test_add::(); - } - - #[test] - fn add_u64() { - test_add::(); - } - - fn test_multiplication() { - for n in 0..100 { - let n = 2 * n + 1; - let m = Montgomery::::new(n); - for x in 0..n { - let m_x = m.to_mod(x); - for y in 0..=x { - let m_y = m.to_mod(y); - assert_eq!((x * y) % n, m.to_u64(m.mul(m_x, m_y))); - } - } - } - } - - #[test] - fn multiplication_u32() { - test_multiplication::(); - } - - #[test] - fn multiplication_u64() { - test_multiplication::(); - } - - fn test_roundtrip() { - for n in 0..100 { - let n = 2 * n + 1; - let m = Montgomery::::new(n); - for x in 0..n { - let x_ = m.to_mod(x); - assert_eq!(x, m.to_u64(x_)); - } - } - } - - #[test] - fn roundtrip_u32() { - test_roundtrip::(); - } - - #[test] - fn roundtrip_u64() { - test_roundtrip::(); - } -} diff --git a/src/uu/factor/src/numeric/traits.rs b/src/uu/factor/src/numeric/traits.rs deleted file mode 100644 index c3528a4ad29..00000000000 --- a/src/uu/factor/src/numeric/traits.rs +++ /dev/null @@ -1,87 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -pub(crate) use num_traits::{ - identities::{One, Zero}, - ops::overflowing::OverflowingAdd, -}; -use num_traits::{ - int::PrimInt, - ops::wrapping::{WrappingMul, WrappingNeg, WrappingSub}, -}; -use std::fmt::{Debug, Display}; - -pub(crate) trait Int: - Display + Debug + PrimInt + OverflowingAdd + WrappingNeg + WrappingSub + WrappingMul -{ - fn as_u64(&self) -> u64; - fn from_u64(n: u64) -> Self; - - #[cfg(debug_assertions)] - fn as_u128(&self) -> u128; -} - -pub(crate) trait DoubleInt: Int { - /// An integer type with twice the width of `Self`. - /// In particular, multiplications (of `Int` values) can be performed in - /// `Self::DoubleWidth` without possibility of overflow. - type DoubleWidth: Int; - - fn as_double_width(self) -> Self::DoubleWidth; - fn from_double_width(n: Self::DoubleWidth) -> Self; -} - -macro_rules! int { - ( $x:ty ) => { - impl Int for $x { - fn as_u64(&self) -> u64 { - *self as u64 - } - fn from_u64(n: u64) -> Self { - n as _ - } - #[cfg(debug_assertions)] - fn as_u128(&self) -> u128 { - *self as u128 - } - } - }; -} -macro_rules! double_int { - ( $x:ty, $y:ty ) => { - int!($x); - impl DoubleInt for $x { - type DoubleWidth = $y; - - fn as_double_width(self) -> $y { - self as _ - } - fn from_double_width(n: $y) -> Self { - n as _ - } - } - }; -} -double_int!(u32, u64); -double_int!(u64, u128); -int!(u128); - -/// Helper macro for instantiating tests over u32 and u64 -#[cfg(test)] -#[macro_export] -macro_rules! parametrized_check { - ( $f:ident ) => { - paste::item! { - #[test] - fn [< $f _ u32 >]() { - $f::() - } - #[test] - fn [< $f _ u64 >]() { - $f::() - } - } - }; -} diff --git a/src/uu/factor/src/rho.rs b/src/uu/factor/src/rho.rs deleted file mode 100644 index 2af0f685564..00000000000 --- a/src/uu/factor/src/rho.rs +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -use rand::distributions::{Distribution, Uniform}; -use rand::rngs::SmallRng; -use rand::{thread_rng, SeedableRng}; -use std::cmp::{max, min}; - -use crate::numeric::*; - -pub(crate) fn find_divisor(n: A) -> u64 { - #![allow(clippy::many_single_char_names)] - let mut rand = { - let range = Uniform::new(1, n.modulus()); - let mut rng = SmallRng::from_rng(&mut thread_rng()).unwrap(); - move || n.to_mod(range.sample(&mut rng)) - }; - - let quadratic = |a, b| move |x| n.add(n.mul(a, n.mul(x, x)), b); - - loop { - let f = quadratic(rand(), rand()); - let mut x = rand(); - let mut y = x; - - loop { - x = f(x); - y = f(f(y)); - let d = { - let _x = n.to_u64(x); - let _y = n.to_u64(y); - gcd(n.modulus(), max(_x, _y) - min(_x, _y)) - }; - if d == n.modulus() { - // Failure, retry with a different quadratic - break; - } else if d > 1 { - return d; - } - } - } -} diff --git a/src/uu/factor/src/table.rs b/src/uu/factor/src/table.rs deleted file mode 100644 index e9b525a3af5..00000000000 --- a/src/uu/factor/src/table.rs +++ /dev/null @@ -1,98 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -// spell-checker: ignore (ToDO) INVS - -use crate::Factors; - -include!(concat!(env!("OUT_DIR"), "/prime_table.rs")); - -pub fn factor(num: &mut u64, factors: &mut Factors) { - for &(prime, inv, ceil) in PRIME_INVERSIONS_U64 { - if *num == 1 { - break; - } - - // inv = prime^-1 mod 2^64 - // ceil = floor((2^64-1) / prime) - // if (num * inv) mod 2^64 <= ceil, then prime divides num - // See https://math.stackexchange.com/questions/1251327/ - // for a nice explanation. - let mut k = 0; - loop { - let x = num.wrapping_mul(inv); - - // While prime divides num - if x <= ceil { - *num = x; - k += 1; - #[cfg(feature = "coz")] - coz::progress!("factor found"); - } else { - if k > 0 { - factors.add(prime, k); - } - break; - } - } - } -} - -pub const CHUNK_SIZE: usize = 8; -pub fn factor_chunk(n_s: &mut [u64; CHUNK_SIZE], f_s: &mut [Factors; CHUNK_SIZE]) { - for &(prime, inv, ceil) in PRIME_INVERSIONS_U64 { - if n_s[0] == 1 && n_s[1] == 1 && n_s[2] == 1 && n_s[3] == 1 { - break; - } - - for (num, factors) in n_s.iter_mut().zip(f_s.iter_mut()) { - if *num == 1 { - continue; - } - let mut k = 0; - loop { - let x = num.wrapping_mul(inv); - - // While prime divides num - if x <= ceil { - *num = x; - k += 1; - } else { - if k > 0 { - factors.add(prime, k); - } - break; - } - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Factors; - use quickcheck::quickcheck; - use rand::{rngs::SmallRng, Rng, SeedableRng}; - - quickcheck! { - fn chunk_vs_iter(seed: u64) -> () { - let mut rng = SmallRng::seed_from_u64(seed); - let mut n_c: [u64; CHUNK_SIZE] = rng.gen(); - let mut f_c: [Factors; CHUNK_SIZE] = rng.gen(); - - let mut n_i = n_c; - let mut f_i = f_c.clone(); - for (n, f) in n_i.iter_mut().zip(f_i.iter_mut()) { - factor(n, f); - } - - factor_chunk(&mut n_c, &mut f_c); - - assert_eq!(n_i, n_c); - assert_eq!(f_i, f_c); - } - } -} diff --git a/src/uu/false/Cargo.toml b/src/uu/false/Cargo.toml index e88fa99783f..4fcbb9f4ca3 100644 --- a/src/uu/false/Cargo.toml +++ b/src/uu/false/Cargo.toml @@ -1,15 +1,19 @@ [package] name = "uu_false" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" -description = "false ~ (uutils) do nothing and fail" -homepage = "https://github.com/uutils/coreutils" +description = "false ~ (uutils) do nothing and fail" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/false" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/false.rs" diff --git a/src/uu/false/src/false.rs b/src/uu/false/src/false.rs index 3ae25e5696d..adf3593ea0d 100644 --- a/src/uu/false/src/false.rs +++ b/src/uu/false/src/false.rs @@ -4,7 +4,7 @@ // file that was distributed with this source code. use clap::{Arg, ArgAction, Command}; use std::{ffi::OsString, io::Write}; -use uucore::error::{set_exit_code, UResult}; +use uucore::error::{UResult, set_exit_code}; use uucore::help_about; const ABOUT: &str = help_about!("false.md"); @@ -28,7 +28,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let error = match e.kind() { clap::error::ErrorKind::DisplayHelp => command.print_help(), clap::error::ErrorKind::DisplayVersion => { - writeln!(std::io::stdout(), "{}", command.render_version()) + write!(std::io::stdout(), "{}", command.render_version()) } _ => Ok(()), }; @@ -36,7 +36,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { // Try to display this error. if let Err(print_fail) = error { // Completely ignore any error here, no more failover and we will fail in any case. - let _ = writeln!(std::io::stderr(), "{}: {}", uucore::util_name(), print_fail); + let _ = writeln!(std::io::stderr(), "{}: {print_fail}", uucore::util_name()); } } @@ -45,7 +45,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(clap::crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) // We provide our own help and version options, to ensure maximum compatibility with GNU. .disable_help_flag(true) diff --git a/src/uu/fmt/Cargo.toml b/src/uu/fmt/Cargo.toml index de415310f2f..65e03b80dfd 100644 --- a/src/uu/fmt/Cargo.toml +++ b/src/uu/fmt/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_fmt" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "fmt ~ (uutils) reformat each paragraph of input" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/fmt" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/fmt.rs" diff --git a/src/uu/fmt/fmt.md b/src/uu/fmt/fmt.md index 5cf4fa6e632..6ed7d3048a5 100644 --- a/src/uu/fmt/fmt.md +++ b/src/uu/fmt/fmt.md @@ -1,7 +1,7 @@ # fmt ``` -fmt [OPTION]... [FILE]... +fmt [-WIDTH] [OPTION]... [FILE]... ``` Reformat paragraphs from input files (or stdin) to stdout. diff --git a/src/uu/fmt/src/fmt.rs b/src/uu/fmt/src/fmt.rs index 4380487814b..8366af6cdba 100644 --- a/src/uu/fmt/src/fmt.rs +++ b/src/uu/fmt/src/fmt.rs @@ -5,12 +5,12 @@ // spell-checker:ignore (ToDO) PSKIP linebreak ostream parasplit tabwidth xanti xprefix -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; use std::fs::File; -use std::io::{stdin, stdout, BufReader, BufWriter, Read, Stdout, Write}; +use std::io::{BufReader, BufWriter, Read, Stdout, Write, stdin, stdout}; use uucore::display::Quotable; -use uucore::error::{FromIo, UResult, USimpleError}; -use uucore::{format_usage, help_about, help_usage, show_warning}; +use uucore::error::{FromIo, UResult, USimpleError, UUsageError}; +use uucore::{format_usage, help_about, help_usage}; use linebreak::break_lines; use parasplit::ParagraphStream; @@ -21,6 +21,10 @@ mod parasplit; const ABOUT: &str = help_about!("fmt.md"); const USAGE: &str = help_usage!("fmt.md"); const MAX_WIDTH: usize = 2500; +const DEFAULT_GOAL: usize = 70; +const DEFAULT_WIDTH: usize = 75; +// by default, goal is 93% of width +const DEFAULT_GOAL_TO_WIDTH_RATIO: usize = 93; mod options { pub const CROWN_MARGIN: &str = "crown-margin"; @@ -36,13 +40,11 @@ mod options { pub const GOAL: &str = "goal"; pub const QUICK: &str = "quick"; pub const TAB_WIDTH: &str = "tab-width"; - pub const FILES: &str = "files"; + pub const FILES_OR_WIDTH: &str = "files"; } -// by default, goal is 93% of width -const DEFAULT_GOAL_TO_WIDTH_RATIO: usize = 93; - pub type FileOrStdReader = BufReader>; + pub struct FmtOptions { crown: bool, tagged: bool, @@ -85,30 +87,55 @@ impl FmtOptions { .get_one::(options::SKIP_PREFIX) .map(String::from); - let width_opt = matches.get_one::(options::WIDTH); - let goal_opt = matches.get_one::(options::GOAL); + let width_opt = extract_width(matches)?; + let goal_opt_str = matches.get_one::(options::GOAL); + let goal_opt = if let Some(goal_str) = goal_opt_str { + match goal_str.parse::() { + Ok(goal) => Some(goal), + Err(_) => { + return Err(USimpleError::new( + 1, + format!("invalid goal: {}", goal_str.quote()), + )); + } + } + } else { + None + }; + let (width, goal) = match (width_opt, goal_opt) { - (Some(&w), Some(&g)) => { + (Some(w), Some(g)) => { if g > w { return Err(USimpleError::new(1, "GOAL cannot be greater than WIDTH.")); } (w, g) } - (Some(&w), None) => { - let g = (w * DEFAULT_GOAL_TO_WIDTH_RATIO / 100).min(w - 3); + (Some(0), None) => { + // Only allow a goal of zero if the width is set to be zero + (0, 0) + } + (Some(w), None) => { + let g = (w * DEFAULT_GOAL_TO_WIDTH_RATIO / 100).max(1); (w, g) } - (None, Some(&g)) => { + (None, Some(g)) => { + if g > DEFAULT_WIDTH { + return Err(USimpleError::new(1, "GOAL cannot be greater than WIDTH.")); + } let w = (g * 100 / DEFAULT_GOAL_TO_WIDTH_RATIO).max(g + 3); (w, g) } - (None, None) => (75, 70), + (None, None) => (DEFAULT_WIDTH, DEFAULT_GOAL), }; + debug_assert!( + width >= goal, + "GOAL {goal} should not be greater than WIDTH {width} when given {width_opt:?} and {goal_opt:?}." + ); if width > MAX_WIDTH { return Err(USimpleError::new( 1, - format!("invalid width: '{}': Numerical result out of range", width), + format!("invalid width: '{width}': Numerical result out of range"), )); } @@ -119,7 +146,7 @@ impl FmtOptions { Err(e) => { return Err(USimpleError::new( 1, - format!("Invalid TABWIDTH specification: {}: {}", s.quote(), e), + format!("Invalid TABWIDTH specification: {}: {e}", s.quote()), )); } }; @@ -133,13 +160,13 @@ impl FmtOptions { crown, tagged, mail, - uniform, - quick, split_only, prefix, xprefix, anti_prefix, xanti_prefix, + uniform, + quick, width, goal, tabwidth, @@ -163,16 +190,21 @@ fn process_file( fmt_opts: &FmtOptions, ostream: &mut BufWriter, ) -> UResult<()> { - let mut fp = match file_name { - "-" => BufReader::new(Box::new(stdin()) as Box), - _ => match File::open(file_name) { - Ok(f) => BufReader::new(Box::new(f) as Box), - Err(e) => { - show_warning!("{}: {}", file_name.maybe_quote(), e); - return Ok(()); + let mut fp = BufReader::new(match file_name { + "-" => Box::new(stdin()) as Box, + _ => { + let f = File::open(file_name) + .map_err_context(|| format!("cannot open {} for reading", file_name.quote()))?; + if f.metadata() + .map_err_context(|| format!("cannot get metadata for {}", file_name.quote()))? + .is_dir() + { + return Err(USimpleError::new(1, "read error".to_string())); } - }, - }; + + Box::new(f) as Box + } + }); let p_stream = ParagraphStream::new(fmt_opts, &mut fp); for para_result in p_stream { @@ -198,14 +230,97 @@ fn process_file( Ok(()) } +/// Extract the file names from the positional arguments, ignoring any negative width in the first +/// position. +/// +/// # Returns +/// A `UResult<()>` with the file names, or an error if one of the file names could not be parsed +/// (e.g., it is given as a negative number not in the first argument and not after a -- +fn extract_files(matches: &ArgMatches) -> UResult> { + let in_first_pos = matches + .index_of(options::FILES_OR_WIDTH) + .is_some_and(|x| x == 1); + let is_neg = |s: &str| s.parse::().is_ok_and(|w| w < 0); + + let files: UResult> = matches + .get_many::(options::FILES_OR_WIDTH) + .into_iter() + .flatten() + .enumerate() + .filter_map(|(i, x)| { + if is_neg(x) { + if in_first_pos && i == 0 { + None + } else { + let first_num = x.chars().nth(1).expect("a negative number should be at least two characters long"); + Some(Err( + UUsageError::new(1, format!("invalid option -- {first_num}; -WIDTH is recognized only when it is the first\noption; use -w N instead")) + )) + } + } else { + Some(Ok(x.clone())) + } + }) + .collect(); + + if files.as_ref().is_ok_and(|f| f.is_empty()) { + Ok(vec!["-".into()]) + } else { + files + } +} + +fn extract_width(matches: &ArgMatches) -> UResult> { + let width_opt = matches.get_one::(options::WIDTH); + if let Some(width_str) = width_opt { + return if let Ok(width) = width_str.parse::() { + Ok(Some(width)) + } else { + Err(USimpleError::new( + 1, + format!("invalid width: {}", width_str.quote()), + )) + }; + } + + if let Some(1) = matches.index_of(options::FILES_OR_WIDTH) { + let width_arg = matches.get_one::(options::FILES_OR_WIDTH).unwrap(); + if let Some(num) = width_arg.strip_prefix('-') { + Ok(num.parse::().ok()) + } else { + // will be treated as a file name + Ok(None) + } + } else { + Ok(None) + } +} + #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let matches = uu_app().try_get_matches_from(args)?; + let args: Vec<_> = args.collect(); - let files: Vec = matches - .get_many::(options::FILES) - .map(|v| v.map(ToString::to_string).collect()) - .unwrap_or(vec!["-".into()]); + // Warn the user if it looks like we're trying to pass a number in the first + // argument with non-numeric characters + if let Some(first_arg) = args.get(1) { + let first_arg = first_arg.to_string_lossy(); + let malformed_number = first_arg.starts_with('-') + && first_arg.chars().nth(1).is_some_and(|c| c.is_ascii_digit()) + && first_arg.chars().skip(2).any(|c| !c.is_ascii_digit()); + if malformed_number { + return Err(USimpleError::new( + 1, + format!( + "invalid width: {}", + first_arg.strip_prefix('-').unwrap().quote() + ), + )); + } + } + + let matches = uu_app().try_get_matches_from(&args)?; + + let files = extract_files(&matches)?; let fmt_opts = FmtOptions::from_matches(&matches)?; @@ -220,10 +335,11 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) + .args_override_self(true) .arg( Arg::new(options::CROWN_MARGIN) .short('c') @@ -323,17 +439,17 @@ pub fn uu_app() -> Command { Arg::new(options::WIDTH) .short('w') .long("width") - .help("Fill output lines up to a maximum of WIDTH columns, default 75.") - .value_name("WIDTH") - .value_parser(clap::value_parser!(usize)), + .help("Fill output lines up to a maximum of WIDTH columns, default 75. This can be specified as a negative number in the first argument.") + // We must accept invalid values if they are overridden later. This is not supported by clap, so accept all strings instead. + .value_name("WIDTH"), ) .arg( Arg::new(options::GOAL) .short('g') .long("goal") - .help("Goal width, default of 93% of WIDTH. Must be less than WIDTH.") - .value_name("GOAL") - .value_parser(clap::value_parser!(usize)), + .help("Goal width, default of 93% of WIDTH. Must be less than or equal to WIDTH.") + // We must accept invalid values if they are overridden later. This is not supported by clap, so accept all strings instead. + .value_name("GOAL"), ) .arg( Arg::new(options::QUICK) @@ -357,8 +473,64 @@ pub fn uu_app() -> Command { .value_name("TABWIDTH"), ) .arg( - Arg::new(options::FILES) + Arg::new(options::FILES_OR_WIDTH) .action(ArgAction::Append) - .value_hint(clap::ValueHint::FilePath), + .value_name("FILES") + .value_hint(clap::ValueHint::FilePath) + .allow_negative_numbers(true), ) } + +#[cfg(test)] +mod tests { + use crate::uu_app; + use crate::{extract_files, extract_width}; + + #[test] + fn parse_negative_width() { + let matches = uu_app() + .try_get_matches_from(vec!["fmt", "-3", "some-file"]) + .unwrap(); + + assert_eq!(extract_files(&matches).unwrap(), vec!["some-file"]); + assert_eq!(extract_width(&matches).ok(), Some(Some(3))); + } + + #[test] + fn parse_width_as_arg() { + let matches = uu_app() + .try_get_matches_from(vec!["fmt", "-w3", "some-file"]) + .unwrap(); + + assert_eq!(extract_files(&matches).unwrap(), vec!["some-file"]); + assert_eq!(extract_width(&matches).ok(), Some(Some(3))); + } + + #[test] + fn parse_no_args() { + let matches = uu_app().try_get_matches_from(vec!["fmt"]).unwrap(); + + assert_eq!(extract_files(&matches).unwrap(), vec!["-"]); + assert_eq!(extract_width(&matches).ok(), Some(None)); + } + + #[test] + fn parse_just_file_name() { + let matches = uu_app() + .try_get_matches_from(vec!["fmt", "some-file"]) + .unwrap(); + + assert_eq!(extract_files(&matches).unwrap(), vec!["some-file"]); + assert_eq!(extract_width(&matches).ok(), Some(None)); + } + + #[test] + fn parse_with_both_widths_positional_first() { + let matches = uu_app() + .try_get_matches_from(vec!["fmt", "-10", "-w3", "some-file"]) + .unwrap(); + + assert_eq!(extract_files(&matches).unwrap(), vec!["some-file"]); + assert_eq!(extract_width(&matches).ok(), Some(Some(3))); + } +} diff --git a/src/uu/fmt/src/linebreak.rs b/src/uu/fmt/src/linebreak.rs index 7393589d0b3..6c34b08088a 100644 --- a/src/uu/fmt/src/linebreak.rs +++ b/src/uu/fmt/src/linebreak.rs @@ -6,10 +6,10 @@ // spell-checker:ignore (ToDO) INFTY MULT accum breakwords linebreak linebreaking linebreaks linelen maxlength minlength nchars ostream overlen parasplit plass posn powf punct signum slen sstart tabwidth tlen underlen winfo wlen wordlen use std::io::{BufWriter, Stdout, Write}; -use std::{cmp, i64, mem}; +use std::{cmp, mem}; -use crate::parasplit::{ParaWords, Paragraph, WordInfo}; use crate::FmtOptions; +use crate::parasplit::{ParaWords, Paragraph, WordInfo}; struct BreakArgs<'a> { opts: &'a FmtOptions, @@ -20,7 +20,7 @@ struct BreakArgs<'a> { ostream: &'a mut BufWriter, } -impl<'a> BreakArgs<'a> { +impl BreakArgs<'_> { fn compute_width(&self, winfo: &WordInfo, posn: usize, fresh: bool) -> usize { if fresh { 0 @@ -164,9 +164,7 @@ fn break_knuth_plass<'a, T: Clone + Iterator>>( // We find identical breakpoints here by comparing addresses of the references. // This is OK because the backing vector is not mutating once we are linebreaking. - let winfo_ptr = winfo as *const _; - let next_break_ptr = next_break as *const _; - if winfo_ptr == next_break_ptr { + if std::ptr::eq(winfo, next_break) { // OK, we found the matching word if break_before { write_newline(args.indent_str, args.ostream)?; @@ -238,8 +236,8 @@ fn find_kp_breakpoints<'a, T: Iterator>>( let mut active_breaks = vec![0]; let mut next_active_breaks = vec![]; - let stretch = (args.opts.width - args.opts.goal) as isize; - let minlength = args.opts.goal - stretch as usize; + let stretch = args.opts.width - args.opts.goal; + let minlength = args.opts.goal.max(stretch + 1) - stretch; let mut new_linebreaks = vec![]; let mut is_sentence_start = false; let mut least_demerits = 0; @@ -300,7 +298,7 @@ fn find_kp_breakpoints<'a, T: Iterator>>( compute_demerits( args.opts.goal as isize - tlen as isize, stretch, - w.word_nchars as isize, + w.word_nchars, active.prev_rat, ) }; @@ -393,7 +391,7 @@ const DR_MULT: f32 = 600.0; // DL_MULT is penalty multiplier for short words at end of line const DL_MULT: f32 = 300.0; -fn compute_demerits(delta_len: isize, stretch: isize, wlen: isize, prev_rat: f32) -> (i64, f32) { +fn compute_demerits(delta_len: isize, stretch: usize, wlen: usize, prev_rat: f32) -> (i64, f32) { // how much stretch are we using? let ratio = if delta_len == 0 { 0.0f32 @@ -419,7 +417,7 @@ fn compute_demerits(delta_len: isize, stretch: isize, wlen: isize, prev_rat: f32 }; // we penalize lines that have very different ratios from previous lines - let bad_delta_r = (DR_MULT * (((ratio - prev_rat) / 2.0).powi(3)).abs()) as i64; + let bad_delta_r = (DR_MULT * ((ratio - prev_rat) / 2.0).powi(3).abs()) as i64; let demerits = i64::pow(1 + bad_linelen + bad_wordlen + bad_delta_r, 2); @@ -440,8 +438,8 @@ fn restart_active_breaks<'a>( } else { // choose the lesser evil: breaking too early, or breaking too late let wlen = w.word_nchars + args.compute_width(w, active.length, active.fresh); - let underlen = (min - active.length) as isize; - let overlen = ((wlen + slen + active.length) - args.opts.width) as isize; + let underlen = min as isize - active.length as isize; + let overlen = (wlen + slen + active.length) as isize - args.opts.width as isize; if overlen > underlen { // break early, put this word on the next line (true, args.indent_len + w.word_nchars) @@ -465,11 +463,7 @@ fn restart_active_breaks<'a>( // Number of spaces to add before a word, based on mode, newline, sentence start. fn compute_slen(uniform: bool, newline: bool, start: bool, punct: bool) -> usize { if uniform || newline { - if start || (newline && punct) { - 2 - } else { - 1 - } + if start || (newline && punct) { 2 } else { 1 } } else { 0 } diff --git a/src/uu/fmt/src/parasplit.rs b/src/uu/fmt/src/parasplit.rs index 1ae8ea34f42..f9da4ad58fd 100644 --- a/src/uu/fmt/src/parasplit.rs +++ b/src/uu/fmt/src/parasplit.rs @@ -73,7 +73,7 @@ pub struct FileLines<'a> { lines: Lines<&'a mut FileOrStdReader>, } -impl<'a> FileLines<'a> { +impl FileLines<'_> { fn new<'b>(opts: &'b FmtOptions, lines: Lines<&'b mut FileOrStdReader>) -> FileLines<'b> { FileLines { opts, lines } } @@ -144,7 +144,7 @@ impl<'a> FileLines<'a> { } } -impl<'a> Iterator for FileLines<'a> { +impl Iterator for FileLines<'_> { type Item = Line; fn next(&mut self) -> Option { @@ -232,7 +232,7 @@ pub struct ParagraphStream<'a> { opts: &'a FmtOptions, } -impl<'a> ParagraphStream<'a> { +impl ParagraphStream<'_> { pub fn new<'b>(opts: &'b FmtOptions, reader: &'b mut FileOrStdReader) -> ParagraphStream<'b> { let lines = FileLines::new(opts, reader.lines()).peekable(); // at the beginning of the file, we might find mail headers @@ -255,9 +255,8 @@ impl<'a> ParagraphStream<'a> { if l_slice.starts_with("From ") { true } else { - let colon_posn = match l_slice.find(':') { - Some(n) => n, - None => return false, + let Some(colon_posn) = l_slice.find(':') else { + return false; }; // header field must be nonzero length @@ -273,7 +272,7 @@ impl<'a> ParagraphStream<'a> { } } -impl<'a> Iterator for ParagraphStream<'a> { +impl Iterator for ParagraphStream<'_> { type Item = Result; #[allow(clippy::cognitive_complexity)] @@ -491,7 +490,7 @@ struct WordSplit<'a> { prev_punct: bool, } -impl<'a> WordSplit<'a> { +impl WordSplit<'_> { fn analyze_tabs(&self, string: &str) -> (Option, usize, Option) { // given a string, determine (length before tab) and (printed length after first tab) // if there are no tabs, beforetab = -1 and aftertab is the printed length @@ -517,7 +516,7 @@ impl<'a> WordSplit<'a> { } } -impl<'a> WordSplit<'a> { +impl WordSplit<'_> { fn new<'b>(opts: &'b FmtOptions, string: &'b str) -> WordSplit<'b> { // wordsplits *must* start at a non-whitespace character let trim_string = string.trim_start(); @@ -560,12 +559,11 @@ impl<'a> Iterator for WordSplit<'a> { // find the start of the next word, and record if we find a tab character let (before_tab, after_tab, word_start) = - match self.analyze_tabs(&self.string[old_position..]) { - (b, a, Some(s)) => (b, a, s + old_position), - (_, _, None) => { - self.position = self.length; - return None; - } + if let (b, a, Some(s)) = self.analyze_tabs(&self.string[old_position..]) { + (b, a, s + old_position) + } else { + self.position = self.length; + return None; }; // find the beginning of the next whitespace diff --git a/src/uu/fold/Cargo.toml b/src/uu/fold/Cargo.toml index 02cf1115ec2..ab6adaba16c 100644 --- a/src/uu/fold/Cargo.toml +++ b/src/uu/fold/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_fold" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "fold ~ (uutils) wrap each line of input" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/fold" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/fold.rs" diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 0223248be27..0aba9c57ee4 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -5,9 +5,9 @@ // spell-checker:ignore (ToDOs) ncount routput -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use std::fs::File; -use std::io::{stdin, BufRead, BufReader, Read}; +use std::io::{BufRead, BufReader, Read, stdin}; use std::path::Path; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError}; @@ -43,7 +43,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { Some(inp_width) => inp_width.parse::().map_err(|e| { USimpleError::new( 1, - format!("illegal width value ({}): {}", inp_width.quote(), e), + format!("illegal width value ({}): {e}", inp_width.quote()), ) })?, None => 80, @@ -59,7 +59,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .override_usage(format_usage(USAGE)) .about(ABOUT) .infer_long_args(true) @@ -99,7 +99,7 @@ pub fn uu_app() -> Command { fn handle_obsolete(args: &[String]) -> (Vec, Option) { for (i, arg) in args.iter().enumerate() { let slice = &arg; - if slice.starts_with('-') && slice.chars().nth(1).map_or(false, |c| c.is_ascii_digit()) { + if slice.starts_with('-') && slice.chars().nth(1).is_some_and(|c| c.is_ascii_digit()) { let mut v = args.to_vec(); v.remove(i); return (v, Some(slice[1..].to_owned())); diff --git a/src/uu/groups/Cargo.toml b/src/uu/groups/Cargo.toml index dac52fc5864..d3cd62d4900 100644 --- a/src/uu/groups/Cargo.toml +++ b/src/uu/groups/Cargo.toml @@ -1,21 +1,25 @@ [package] name = "uu_groups" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "groups ~ (uutils) display group memberships for USERNAME" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/groups" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/groups.rs" [dependencies] clap = { workspace = true } +thiserror = { workspace = true } uucore = { workspace = true, features = ["entries", "process"] } [[bin]] diff --git a/src/uu/groups/src/groups.rs b/src/uu/groups/src/groups.rs index 0f0dfce804a..6f7fbf5fed2 100644 --- a/src/uu/groups/src/groups.rs +++ b/src/uu/groups/src/groups.rs @@ -2,26 +2,18 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// -// ============================================================================ -// Test suite summary for GNU coreutils 8.32.162-4eda -// ============================================================================ -// PASS: tests/misc/groups-dash.sh -// PASS: tests/misc/groups-process-all.sh -// PASS: tests/misc/groups-version.sh // spell-checker:ignore (ToDO) passwd -use std::error::Error; -use std::fmt::Display; +use thiserror::Error; use uucore::{ display::Quotable, - entries::{get_groups_gnu, gid2grp, Locate, Passwd}, + entries::{Locate, Passwd, get_groups_gnu, gid2grp}, error::{UError, UResult}, format_usage, help_about, help_usage, show, }; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; mod options { pub const USERS: &str = "USERNAME"; @@ -29,26 +21,20 @@ mod options { const ABOUT: &str = help_about!("groups.md"); const USAGE: &str = help_usage!("groups.md"); -#[derive(Debug)] +#[derive(Debug, Error)] enum GroupsError { + #[error("failed to fetch groups")] GetGroupsFailed, + + #[error("cannot find name for group ID {0}")] GroupNotFound(u32), + + #[error("{user}: no such user", user = .0.quote())] UserNotFound(String), } -impl Error for GroupsError {} impl UError for GroupsError {} -impl Display for GroupsError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::GetGroupsFailed => write!(f, "failed to fetch groups"), - Self::GroupNotFound(gid) => write!(f, "cannot find name for group ID {gid}"), - Self::UserNotFound(user) => write!(f, "{}: no such user", user.quote()), - } - } -} - fn infallible_gid2grp(gid: &u32) -> String { match gid2grp(*gid) { Ok(grp) => grp, @@ -70,9 +56,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { .unwrap_or_default(); if users.is_empty() { - let gids = match get_groups_gnu(None) { - Ok(v) => v, - Err(_) => return Err(GroupsError::GetGroupsFailed.into()), + let Ok(gids) = get_groups_gnu(None) else { + return Err(GroupsError::GetGroupsFailed.into()); }; let groups: Vec = gids.iter().map(infallible_gid2grp).collect(); println!("{}", groups.join(" ")); @@ -83,7 +68,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { match Passwd::locate(user.as_str()) { Ok(p) => { let groups: Vec = p.belongs_to().iter().map(infallible_gid2grp).collect(); - println!("{} : {}", user, groups.join(" ")); + println!("{user} : {}", groups.join(" ")); } Err(_) => { // The `show!()` macro sets the global exit code for the program. @@ -96,7 +81,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) diff --git a/src/uu/hashsum/Cargo.toml b/src/uu/hashsum/Cargo.toml index a20fe0d7826..ff1c7de1c10 100644 --- a/src/uu/hashsum/Cargo.toml +++ b/src/uu/hashsum/Cargo.toml @@ -1,22 +1,25 @@ [package] name = "uu_hashsum" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "hashsum ~ (uutils) display or check input digests" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/hashsum" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/hashsum.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true, features = ["sum"] } +uucore = { workspace = true, features = ["checksum", "sum"] } memchr = { workspace = true } regex = { workspace = true } hex = { workspace = true } diff --git a/src/uu/hashsum/hashsum.md b/src/uu/hashsum/hashsum.md index a632eedf3d9..35ec840a39c 100644 --- a/src/uu/hashsum/hashsum.md +++ b/src/uu/hashsum/hashsum.md @@ -1,7 +1,7 @@ # hashsum ``` -hashsum [OPTIONS] [FILE]... +hashsum -- [OPTIONS]... [FILE]... ``` Compute and check message digests. diff --git a/src/uu/hashsum/src/hashsum.rs b/src/uu/hashsum/src/hashsum.rs index 5e439853a58..cd8ca912df5 100644 --- a/src/uu/hashsum/src/hashsum.rs +++ b/src/uu/hashsum/src/hashsum.rs @@ -5,28 +5,28 @@ // spell-checker:ignore (ToDO) algo, algoname, regexes, nread, nonames -use clap::builder::ValueParser; -use clap::crate_version; use clap::ArgAction; +use clap::builder::ValueParser; +use clap::value_parser; use clap::{Arg, ArgMatches, Command}; -use hex::encode; -use regex::Captures; -use regex::Regex; -use std::cmp::Ordering; -use std::error::Error; use std::ffi::{OsStr, OsString}; use std::fs::File; -use std::io::{self, stdin, BufRead, BufReader, Read}; +use std::io::{BufReader, Read, stdin}; use std::iter; use std::num::ParseIntError; use std::path::Path; -use uucore::error::USimpleError; -use uucore::error::{FromIo, UError, UResult}; -use uucore::sum::{ - Blake2b, Blake3, Digest, DigestWriter, Md5, Sha1, Sha224, Sha256, Sha384, Sha3_224, Sha3_256, - Sha3_384, Sha3_512, Sha512, Shake128, Shake256, -}; -use uucore::{display::Quotable, show_warning}; +use uucore::checksum::ChecksumError; +use uucore::checksum::ChecksumOptions; +use uucore::checksum::ChecksumVerbose; +use uucore::checksum::HashAlgorithm; +use uucore::checksum::calculate_blake2b_length; +use uucore::checksum::create_sha3; +use uucore::checksum::detect_algo; +use uucore::checksum::digest_reader; +use uucore::checksum::escape_filename; +use uucore::checksum::perform_checksum_validation; +use uucore::error::{FromIo, UResult}; +use uucore::sum::{Digest, Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}; use uucore::{format_usage, help_about, help_usage}; const NAME: &str = "hashsum"; @@ -37,172 +37,16 @@ struct Options { algoname: &'static str, digest: Box, binary: bool, - check: bool, + //check: bool, tag: bool, nonames: bool, - status: bool, - quiet: bool, - strict: bool, - warn: bool, + //status: bool, + //quiet: bool, + //strict: bool, + //warn: bool, output_bits: usize, zero: bool, -} - -/// Creates a Blake2b hasher instance based on the specified length argument. -/// -/// # Returns -/// -/// Returns a UResult of a tuple containing the algorithm name, the hasher instance, and -/// the output length in bits or an Err if the length is not a multiple of 8 or if it is -/// greater than 512. -fn create_blake2b(matches: &ArgMatches) -> UResult<(&'static str, Box, usize)> { - match matches.get_one::("length") { - Some(0) | None => Ok(("BLAKE2", Box::new(Blake2b::new()) as Box, 512)), - Some(length_in_bits) => { - if *length_in_bits > 512 { - return Err(USimpleError::new( - 1, - "Invalid length (maximum digest length is 512 bits)", - )); - } - - if length_in_bits % 8 == 0 { - let length_in_bytes = length_in_bits / 8; - Ok(( - "BLAKE2", - Box::new(Blake2b::with_output_bytes(length_in_bytes)), - *length_in_bits, - )) - } else { - Err(USimpleError::new( - 1, - "Invalid length (expected a multiple of 8)", - )) - } - } - } -} - -/// Creates a SHA3 hasher instance based on the specified bits argument. -/// -/// # Returns -/// -/// Returns a UResult of a tuple containing the algorithm name, the hasher instance, and -/// the output length in bits or an Err if an unsupported output size is provided, or if -/// the `--bits` flag is missing. -fn create_sha3(matches: &ArgMatches) -> UResult<(&'static str, Box, usize)> { - match matches.get_one::("bits") { - Some(224) => Ok(( - "SHA3-224", - Box::new(Sha3_224::new()) as Box, - 224, - )), - Some(256) => Ok(( - "SHA3-256", - Box::new(Sha3_256::new()) as Box, - 256, - )), - Some(384) => Ok(( - "SHA3-384", - Box::new(Sha3_384::new()) as Box, - 384, - )), - Some(512) => Ok(( - "SHA3-512", - Box::new(Sha3_512::new()) as Box, - 512, - )), - Some(_) => Err(USimpleError::new( - 1, - "Invalid output size for SHA3 (expected 224, 256, 384, or 512)", - )), - None => Err(USimpleError::new(1, "--bits required for SHA3")), - } -} - -/// Creates a SHAKE-128 hasher instance based on the specified bits argument. -/// -/// # Returns -/// -/// Returns a UResult of a tuple containing the algorithm name, the hasher instance, and -/// the output length in bits, or an Err if `--bits` flag is missing. -fn create_shake128(matches: &ArgMatches) -> UResult<(&'static str, Box, usize)> { - match matches.get_one::("bits") { - Some(bits) => Ok(( - "SHAKE128", - Box::new(Shake128::new()) as Box, - *bits, - )), - None => Err(USimpleError::new(1, "--bits required for SHAKE-128")), - } -} - -/// Creates a SHAKE-256 hasher instance based on the specified bits argument. -/// -/// # Returns -/// -/// Returns a UResult of a tuple containing the algorithm name, the hasher instance, and -/// the output length in bits, or an Err if the `--bits` flag is missing. -fn create_shake256(matches: &ArgMatches) -> UResult<(&'static str, Box, usize)> { - match matches.get_one::("bits") { - Some(bits) => Ok(( - "SHAKE256", - Box::new(Shake256::new()) as Box, - *bits, - )), - None => Err(USimpleError::new(1, "--bits required for SHAKE-256")), - } -} - -/// Detects the hash algorithm from the program name or command-line arguments. -/// -/// # Arguments -/// -/// * `program` - A string slice containing the program name. -/// * `matches` - A reference to the `ArgMatches` object containing the command-line arguments. -/// -/// # Returns -/// -/// Returns a UResult of a tuple containing the algorithm name, the hasher instance, and -/// the output length in bits, or an Err if a matching algorithm is not found. -fn detect_algo( - program: &str, - matches: &ArgMatches, -) -> UResult<(&'static str, Box, usize)> { - match program { - "md5sum" => Ok(("MD5", Box::new(Md5::new()) as Box, 128)), - "sha1sum" => Ok(("SHA1", Box::new(Sha1::new()) as Box, 160)), - "sha224sum" => Ok(("SHA224", Box::new(Sha224::new()) as Box, 224)), - "sha256sum" => Ok(("SHA256", Box::new(Sha256::new()) as Box, 256)), - "sha384sum" => Ok(("SHA384", Box::new(Sha384::new()) as Box, 384)), - "sha512sum" => Ok(("SHA512", Box::new(Sha512::new()) as Box, 512)), - "b2sum" => create_blake2b(matches), - "b3sum" => Ok(("BLAKE3", Box::new(Blake3::new()) as Box, 256)), - "sha3sum" => create_sha3(matches), - "sha3-224sum" => Ok(( - "SHA3-224", - Box::new(Sha3_224::new()) as Box, - 224, - )), - "sha3-256sum" => Ok(( - "SHA3-256", - Box::new(Sha3_256::new()) as Box, - 256, - )), - "sha3-384sum" => Ok(( - "SHA3-384", - Box::new(Sha3_384::new()) as Box, - 384, - )), - "sha3-512sum" => Ok(( - "SHA3-512", - Box::new(Sha3_512::new()) as Box, - 512, - )), - "shake128sum" => create_shake128(matches), - "shake256sum" => create_shake256(matches), - _ => create_algorithm_from_flags(matches), - } + //ignore_missing: bool, } /// Creates a hasher instance based on the command-line flags. @@ -217,85 +61,99 @@ fn detect_algo( /// the output length in bits or an Err if multiple hash algorithms are specified or if a /// required flag is missing. #[allow(clippy::cognitive_complexity)] -fn create_algorithm_from_flags( - matches: &ArgMatches, -) -> UResult<(&'static str, Box, usize)> { - let mut alg: Option> = None; - let mut name: &'static str = ""; - let mut output_bits = 0; - let mut set_or_err = |n, val, bits| { - if alg.is_some() { - return Err(USimpleError::new( - 1, - "You cannot combine multiple hash algorithms!", - )); - }; - name = n; - alg = Some(val); - output_bits = bits; +fn create_algorithm_from_flags(matches: &ArgMatches) -> UResult { + let mut alg: Option = None; + let mut set_or_err = |new_alg: HashAlgorithm| -> UResult<()> { + if alg.is_some() { + return Err(ChecksumError::CombineMultipleAlgorithms.into()); + } + alg = Some(new_alg); Ok(()) }; if matches.get_flag("md5") { - set_or_err("MD5", Box::new(Md5::new()), 128)?; + set_or_err(detect_algo("md5sum", None)?)?; } if matches.get_flag("sha1") { - set_or_err("SHA1", Box::new(Sha1::new()), 160)?; + set_or_err(detect_algo("sha1sum", None)?)?; } if matches.get_flag("sha224") { - set_or_err("SHA224", Box::new(Sha224::new()), 224)?; + set_or_err(detect_algo("sha224sum", None)?)?; } if matches.get_flag("sha256") { - set_or_err("SHA256", Box::new(Sha256::new()), 256)?; + set_or_err(detect_algo("sha256sum", None)?)?; } if matches.get_flag("sha384") { - set_or_err("SHA384", Box::new(Sha384::new()), 384)?; + set_or_err(detect_algo("sha384sum", None)?)?; } if matches.get_flag("sha512") { - set_or_err("SHA512", Box::new(Sha512::new()), 512)?; + set_or_err(detect_algo("sha512sum", None)?)?; } if matches.get_flag("b2sum") { - set_or_err("BLAKE2", Box::new(Blake2b::new()), 512)?; + set_or_err(detect_algo("b2sum", None)?)?; } if matches.get_flag("b3sum") { - set_or_err("BLAKE3", Box::new(Blake3::new()), 256)?; + set_or_err(detect_algo("b3sum", None)?)?; } if matches.get_flag("sha3") { - let (n, val, bits) = create_sha3(matches)?; - set_or_err(n, val, bits)?; + let bits = matches.get_one::("bits").copied(); + set_or_err(create_sha3(bits)?)?; } if matches.get_flag("sha3-224") { - set_or_err("SHA3-224", Box::new(Sha3_224::new()), 224)?; + set_or_err(HashAlgorithm { + name: "SHA3-224", + create_fn: Box::new(|| Box::new(Sha3_224::new())), + bits: 224, + })?; } if matches.get_flag("sha3-256") { - set_or_err("SHA3-256", Box::new(Sha3_256::new()), 256)?; + set_or_err(HashAlgorithm { + name: "SHA3-256", + create_fn: Box::new(|| Box::new(Sha3_256::new())), + bits: 256, + })?; } if matches.get_flag("sha3-384") { - set_or_err("SHA3-384", Box::new(Sha3_384::new()), 384)?; + set_or_err(HashAlgorithm { + name: "SHA3-384", + create_fn: Box::new(|| Box::new(Sha3_384::new())), + bits: 384, + })?; } if matches.get_flag("sha3-512") { - set_or_err("SHA3-512", Box::new(Sha3_512::new()), 512)?; + set_or_err(HashAlgorithm { + name: "SHA3-512", + create_fn: Box::new(|| Box::new(Sha3_512::new())), + bits: 512, + })?; } if matches.get_flag("shake128") { match matches.get_one::("bits") { - Some(bits) => set_or_err("SHAKE128", Box::new(Shake128::new()), *bits)?, - None => return Err(USimpleError::new(1, "--bits required for SHAKE-128")), + Some(bits) => set_or_err(HashAlgorithm { + name: "SHAKE128", + create_fn: Box::new(|| Box::new(Shake128::new())), + bits: *bits, + })?, + None => return Err(ChecksumError::BitsRequiredForShake128.into()), }; } if matches.get_flag("shake256") { match matches.get_one::("bits") { - Some(bits) => set_or_err("SHAKE256", Box::new(Shake256::new()), *bits)?, - None => return Err(USimpleError::new(1, "--bits required for SHAKE-256")), + Some(bits) => set_or_err(HashAlgorithm { + name: "SHAKE256", + create_fn: Box::new(|| Box::new(Shake256::new())), + bits: *bits, + })?, + None => return Err(ChecksumError::BitsRequiredForShake256.into()), }; } - let alg = match alg { - Some(a) => a, - None => return Err(USimpleError::new(1, "You must specify hash algorithm!")), - }; + if alg.is_none() { + return Err(ChecksumError::NeedAlgorithmToHash.into()); + } - Ok((name, alg, output_bits)) + Ok(alg.unwrap()) } // TODO: return custom error type @@ -317,7 +175,7 @@ pub fn uumain(mut args: impl uucore::Args) -> UResult<()> { // Default binary in Windows, text mode otherwise let binary_flag_default = cfg!(windows); - let command = uu_app(&binary_name); + let (command, is_hashsum_bin) = uu_app(&binary_name); // FIXME: this should use try_get_matches_from() and crash!(), but at the moment that just // causes "error: " to be printed twice (once from crash!() and once from clap). With @@ -325,7 +183,22 @@ pub fn uumain(mut args: impl uucore::Args) -> UResult<()> { // least somewhat better from a user's perspective. let matches = command.try_get_matches_from(args)?; - let (name, algo, bits) = detect_algo(&binary_name, &matches)?; + let input_length: Option<&usize> = if binary_name == "b2sum" { + matches.get_one::(options::LENGTH) + } else { + None + }; + + let length = match input_length { + Some(length) => calculate_blake2b_length(*length)?, + None => None, + }; + + let algo = if is_hashsum_bin { + create_algorithm_from_flags(&matches)? + } else { + detect_algo(&binary_name, length)? + }; let binary = if matches.get_flag("binary") { true @@ -335,74 +208,144 @@ pub fn uumain(mut args: impl uucore::Args) -> UResult<()> { binary_flag_default }; let check = matches.get_flag("check"); - let tag = matches.get_flag("tag"); - let nonames = *matches - .try_get_one("no-names") - .unwrap_or(None) - .unwrap_or(&false); let status = matches.get_flag("status"); let quiet = matches.get_flag("quiet") || status; let strict = matches.get_flag("strict"); let warn = matches.get_flag("warn") && !status; + let ignore_missing = matches.get_flag("ignore-missing"); + + if ignore_missing && !check { + // --ignore-missing needs -c + return Err(ChecksumError::IgnoreNotCheck.into()); + } + + if check { + // on Windows, allow --binary/--text to be used with --check + // and keep the behavior of defaulting to binary + #[cfg(not(windows))] + let binary = { + let text_flag = matches.get_flag("text"); + let binary_flag = matches.get_flag("binary"); + + if binary_flag || text_flag { + return Err(ChecksumError::BinaryTextConflict.into()); + } + + false + }; + + // Execute the checksum validation based on the presence of files or the use of stdin + // Determine the source of input: a list of files or stdin. + let input = matches.get_many::(options::FILE).map_or_else( + || iter::once(OsStr::new("-")).collect::>(), + |files| files.map(OsStr::new).collect::>(), + ); + + let verbose = ChecksumVerbose::new(status, quiet, warn); + + let opts = ChecksumOptions { + binary, + ignore_missing, + strict, + verbose, + }; + + // Execute the checksum validation + return perform_checksum_validation( + input.iter().copied(), + Some(algo.name), + Some(algo.bits), + opts, + ); + } else if quiet { + return Err(ChecksumError::QuietNotCheck.into()); + } else if strict { + return Err(ChecksumError::StrictNotCheck.into()); + } + + let nonames = *matches + .try_get_one("no-names") + .unwrap_or(None) + .unwrap_or(&false); let zero = matches.get_flag("zero"); let opts = Options { - algoname: name, - digest: algo, - output_bits: bits, + algoname: algo.name, + digest: (algo.create_fn)(), + output_bits: algo.bits, binary, - check, - tag, + tag: matches.get_flag("tag"), nonames, - status, - quiet, - strict, - warn, + //status, + //quiet, + //warn, zero, + //ignore_missing, }; - match matches.get_many::("FILE") { + // Show the hashsum of the input + match matches.get_many::(options::FILE) { Some(files) => hashsum(opts, files.map(|f| f.as_os_str())), None => hashsum(opts, iter::once(OsStr::new("-"))), } } +mod options { + //pub const ALGORITHM: &str = "algorithm"; + pub const FILE: &str = "file"; + //pub const UNTAGGED: &str = "untagged"; + pub const TAG: &str = "tag"; + pub const LENGTH: &str = "length"; + //pub const RAW: &str = "raw"; + //pub const BASE64: &str = "base64"; + pub const CHECK: &str = "check"; + pub const STRICT: &str = "strict"; + pub const TEXT: &str = "text"; + pub const BINARY: &str = "binary"; + pub const STATUS: &str = "status"; + pub const WARN: &str = "warn"; + pub const QUIET: &str = "quiet"; +} + pub fn uu_app_common() -> Command { #[cfg(windows)] - const BINARY_HELP: &str = "read in binary mode (default)"; + const BINARY_HELP: &str = "read or check in binary mode (default)"; #[cfg(not(windows))] const BINARY_HELP: &str = "read in binary mode"; #[cfg(windows)] - const TEXT_HELP: &str = "read in text mode"; + const TEXT_HELP: &str = "read or check in text mode"; #[cfg(not(windows))] const TEXT_HELP: &str = "read in text mode (default)"; Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) + .args_override_self(true) .arg( - Arg::new("binary") + Arg::new(options::BINARY) .short('b') .long("binary") .help(BINARY_HELP) .action(ArgAction::SetTrue), ) .arg( - Arg::new("check") + Arg::new(options::CHECK) .short('c') .long("check") .help("read hashsums from the FILEs and check them") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .conflicts_with("tag"), ) .arg( - Arg::new("tag") + Arg::new(options::TAG) .long("tag") .help("create a BSD-style checksum") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .conflicts_with("text"), ) .arg( - Arg::new("text") + Arg::new(options::TEXT) .short('t') .long("text") .help(TEXT_HELP) @@ -410,31 +353,40 @@ pub fn uu_app_common() -> Command { .action(ArgAction::SetTrue), ) .arg( - Arg::new("quiet") + Arg::new(options::QUIET) .short('q') - .long("quiet") + .long(options::QUIET) .help("don't print OK for each successfully verified file") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with_all([options::STATUS, options::WARN]), ) .arg( - Arg::new("status") + Arg::new(options::STATUS) .short('s') .long("status") .help("don't output anything, status code shows success") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with_all([options::QUIET, options::WARN]), ) .arg( - Arg::new("strict") + Arg::new(options::STRICT) .long("strict") .help("exit non-zero for improperly formatted checksum lines") .action(ArgAction::SetTrue), ) .arg( - Arg::new("warn") + Arg::new("ignore-missing") + .long("ignore-missing") + .help("don't fail or report status for missing files") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::WARN) .short('w') .long("warn") .help("warn about improperly formatted checksum lines") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with_all([options::QUIET, options::STATUS]), ) .arg( Arg::new("zero") @@ -444,10 +396,10 @@ pub fn uu_app_common() -> Command { .action(ArgAction::SetTrue), ) .arg( - Arg::new("FILE") + Arg::new(options::FILE) .index(1) .action(ArgAction::Append) - .value_name("FILE") + .value_name(options::FILE) .value_hint(clap::ValueHint::FilePath) .value_parser(ValueParser::os_string()), ) @@ -459,12 +411,16 @@ pub fn uu_app_length() -> Command { fn uu_app_opt_length(command: Command) -> Command { command.arg( - Arg::new("length") + Arg::new(options::LENGTH) + .long(options::LENGTH) + .value_parser(value_parser!(usize)) .short('l') - .long("length") - .help("digest length in bits; must not exceed the max for the blake2 algorithm (512) and must be a multiple of 8") - .value_name("BITS") - .value_parser(parse_bit_num), + .help( + "digest length in bits; must not exceed the max for the blake2 algorithm \ + and must be a multiple of 8", + ) + .overrides_with(options::LENGTH) + .action(ArgAction::Set), ) } @@ -536,43 +492,25 @@ pub fn uu_app_custom() -> Command { // hashsum is handled differently in build.rs, therefore this is not the same // as in other utilities. -fn uu_app(binary_name: &str) -> Command { +fn uu_app(binary_name: &str) -> (Command, bool) { match binary_name { // These all support the same options. "md5sum" | "sha1sum" | "sha224sum" | "sha256sum" | "sha384sum" | "sha512sum" => { - uu_app_common() + (uu_app_common(), false) } // b2sum supports the md5sum options plus -l/--length. - "b2sum" => uu_app_length(), + "b2sum" => (uu_app_length(), false), // These have never been part of GNU Coreutils, but can function with the same // options as md5sum. - "sha3-224sum" | "sha3-256sum" | "sha3-384sum" | "sha3-512sum" => uu_app_common(), + "sha3-224sum" | "sha3-256sum" | "sha3-384sum" | "sha3-512sum" => (uu_app_common(), false), // These have never been part of GNU Coreutils, and require an additional --bits // option to specify their output size. - "sha3sum" | "shake128sum" | "shake256sum" => uu_app_bits(), + "sha3sum" | "shake128sum" | "shake256sum" => (uu_app_bits(), false), // b3sum has never been part of GNU Coreutils, and has a --no-names option in // addition to the b2sum options. - "b3sum" => uu_app_b3sum(), + "b3sum" => (uu_app_b3sum(), false), // We're probably just being called as `hashsum`, so give them everything. - _ => uu_app_custom(), - } -} - -#[derive(Debug)] -enum HashsumError { - InvalidRegex, - InvalidFormat, -} - -impl Error for HashsumError {} -impl UError for HashsumError {} - -impl std::fmt::Display for HashsumError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::InvalidRegex => write!(f, "invalid regular expression"), - Self::InvalidFormat => Ok(()), - } + _ => (uu_app_custom(), true), } } @@ -581,9 +519,6 @@ fn hashsum<'a, I>(mut options: Options, files: I) -> UResult<()> where I: Iterator, { - let mut bad_format = 0; - let mut failed_cksum = 0; - let mut failed_open_file = 0; let binary_marker = if options.binary { "*" } else { " " }; for filename in files { let filename = Path::new(filename); @@ -598,231 +533,40 @@ where File::open(filename).map_err_context(|| "failed to open file".to_string())?; Box::new(file_buf) as Box }); - if options.check { - // Set up Regexes for line validation and parsing - // - // First, we compute the number of bytes we expect to be in - // the digest string. If the algorithm has a variable number - // of output bits, then we use the `+` modifier in the - // regular expression, otherwise we use the `{n}` modifier, - // where `n` is the number of bytes. - let bytes = options.digest.output_bits() / 4; - let bytes_marker = if bytes > 0 { - format!("{{{bytes}}}") - } else { - "+".to_string() - }; - // BSD reversed mode format is similar to the default mode, but doesn’t use a character to distinguish binary and text modes. - let mut bsd_reversed = None; - - /// Creates a Regex for parsing lines based on the given format. - /// The default value of `gnu_re` created with this function has to be recreated - /// after the initial line has been parsed, as this line dictates the format - /// for the rest of them, and mixing of formats is disallowed. - fn gnu_re_template( - bytes_marker: &str, - format_marker: &str, - ) -> Result { - Regex::new(&format!( - r"^(?P[a-fA-F0-9]{bytes_marker}) {format_marker}(?P.*)" - )) - .map_err(|_| HashsumError::InvalidRegex) - } - let mut gnu_re = gnu_re_template(&bytes_marker, r"(?P[ \*])?")?; - let bsd_re = Regex::new(&format!( - r"^{algorithm} \((?P.*)\) = (?P[a-fA-F0-9]{digest_size})", - algorithm = options.algoname, - digest_size = bytes_marker, - )) - .map_err(|_| HashsumError::InvalidRegex)?; - - fn handle_captures( - caps: &Captures, - bytes_marker: &str, - bsd_reversed: &mut Option, - gnu_re: &mut Regex, - ) -> Result<(String, String, bool), HashsumError> { - if bsd_reversed.is_none() { - let is_bsd_reversed = caps.name("binary").is_none(); - let format_marker = if is_bsd_reversed { - "" - } else { - r"(?P[ \*])" - } - .to_string(); - - *bsd_reversed = Some(is_bsd_reversed); - *gnu_re = gnu_re_template(bytes_marker, &format_marker)?; - } - - Ok(( - caps.name("fileName").unwrap().as_str().to_string(), - caps.name("digest").unwrap().as_str().to_ascii_lowercase(), - if *bsd_reversed == Some(false) { - caps.name("binary").unwrap().as_str() == "*" - } else { - false - }, - )) - } - let buffer = file; - for (i, maybe_line) in buffer.lines().enumerate() { - let line = match maybe_line { - Ok(l) => l, - Err(e) => return Err(e.map_err_context(|| "failed to read file".to_string())), - }; - let (ck_filename, sum, binary_check) = match gnu_re.captures(&line) { - Some(caps) => { - handle_captures(&caps, &bytes_marker, &mut bsd_reversed, &mut gnu_re)? - } - None => match bsd_re.captures(&line) { - Some(caps) => ( - caps.name("fileName").unwrap().as_str().to_string(), - caps.name("digest").unwrap().as_str().to_ascii_lowercase(), - true, - ), - None => { - bad_format += 1; - if options.strict { - return Err(HashsumError::InvalidFormat.into()); - } - if options.warn { - show_warning!( - "{}: {}: improperly formatted {} checksum line", - filename.maybe_quote(), - i + 1, - options.algoname - ); - } - continue; - } - }, - }; - let f = match File::open(ck_filename.clone()) { - Err(_) => { - failed_open_file += 1; - println!( - "{}: {}: No such file or directory", - uucore::util_name(), - ck_filename - ); - println!("{ck_filename}: FAILED open or read"); - continue; - } - Ok(file) => file, - }; - let mut ckf = BufReader::new(Box::new(f) as Box); - let real_sum = digest_reader( - &mut options.digest, - &mut ckf, - binary_check, - options.output_bits, - ) - .map_err_context(|| "failed to read input".to_string())? - .to_ascii_lowercase(); - // FIXME: Filenames with newlines should be treated specially. - // GNU appears to replace newlines by \n and backslashes by - // \\ and prepend a backslash (to the hash or filename) if it did - // this escaping. - // Different sorts of output (checking vs outputting hashes) may - // handle this differently. Compare carefully to GNU. - // If you can, try to preserve invalid unicode using OsStr(ing)Ext - // and display it using uucore::display::print_verbatim(). This is - // easier (and more important) on Unix than on Windows. - if sum == real_sum { - if !options.quiet { - println!("{ck_filename}: OK"); - } + let (sum, _) = digest_reader( + &mut options.digest, + &mut file, + options.binary, + options.output_bits, + ) + .map_err_context(|| "failed to read input".to_string())?; + let (escaped_filename, prefix) = escape_filename(filename); + if options.tag { + if options.algoname == "blake2b" { + if options.digest.output_bits() == 512 { + println!("BLAKE2b ({escaped_filename}) = {sum}"); } else { - if !options.status { - println!("{ck_filename}: FAILED"); - } - failed_cksum += 1; + // special case for BLAKE2b with non-default output length + println!( + "BLAKE2b-{} ({escaped_filename}) = {sum}", + options.digest.output_bits() + ); } - } - } else { - let sum = digest_reader( - &mut options.digest, - &mut file, - options.binary, - options.output_bits, - ) - .map_err_context(|| "failed to read input".to_string())?; - if options.tag { - println!("{} ({}) = {}", options.algoname, filename.display(), sum); - } else if options.nonames { - println!("{sum}"); - } else if options.zero { - print!("{} {}{}\0", sum, binary_marker, filename.display()); } else { - let (filename, has_prefix) = escape_filename(filename); - let prefix = if has_prefix { "\\" } else { "" }; - println!("{}{} {}{}", prefix, sum, binary_marker, filename); + println!( + "{prefix}{} ({escaped_filename}) = {sum}", + options.algoname.to_ascii_uppercase() + ); } + } else if options.nonames { + println!("{sum}"); + } else if options.zero { + // with zero, we don't escape the filename + print!("{sum} {binary_marker}{}\0", filename.display()); + } else { + println!("{prefix}{sum} {binary_marker}{escaped_filename}"); } } - if !options.status { - match bad_format.cmp(&1) { - Ordering::Equal => show_warning!("{} line is improperly formatted", bad_format), - Ordering::Greater => show_warning!("{} lines are improperly formatted", bad_format), - Ordering::Less => {} - }; - if failed_cksum > 0 { - show_warning!("{} computed checksum did NOT match", failed_cksum); - } - match failed_open_file.cmp(&1) { - Ordering::Equal => show_warning!("{} listed file could not be read", failed_open_file), - Ordering::Greater => { - show_warning!("{} listed files could not be read", failed_open_file); - } - Ordering::Less => {} - } - } - Ok(()) } - -fn escape_filename(filename: &Path) -> (String, bool) { - let original = filename.as_os_str().to_string_lossy(); - let escaped = original - .replace('\\', "\\\\") - .replace('\n', "\\n") - .replace('\r', "\\r"); - let has_changed = escaped != original; - (escaped, has_changed) -} - -fn digest_reader( - digest: &mut Box, - reader: &mut BufReader, - binary: bool, - output_bits: usize, -) -> io::Result { - digest.reset(); - - // Read bytes from `reader` and write those bytes to `digest`. - // - // If `binary` is `false` and the operating system is Windows, then - // `DigestWriter` replaces "\r\n" with "\n" before it writes the - // bytes into `digest`. Otherwise, it just inserts the bytes as-is. - // - // In order to support replacing "\r\n", we must call `finalize()` - // in order to support the possibility that the last character read - // from the reader was "\r". (This character gets buffered by - // `DigestWriter` and only written if the following character is - // "\n". But when "\r" is the last character read, we need to force - // it to be written.) - let mut digest_writer = DigestWriter::new(digest, binary); - std::io::copy(reader, &mut digest_writer)?; - digest_writer.finalize(); - - if digest.output_bits() > 0 { - Ok(digest.result_str()) - } else { - // Assume it's SHAKE. result_str() doesn't work with shake (as of 8/30/2016) - let mut bytes = vec![0; (output_bits + 7) / 8]; - digest.hash_finalize(&mut bytes); - Ok(encode(bytes)) - } -} diff --git a/src/uu/head/Cargo.toml b/src/uu/head/Cargo.toml index 45e51b881b2..a812bac3160 100644 --- a/src/uu/head/Cargo.toml +++ b/src/uu/head/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_head" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "head ~ (uutils) display the first lines of input" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/head" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/head.rs" @@ -17,7 +20,13 @@ path = "src/head.rs" [dependencies] clap = { workspace = true } memchr = { workspace = true } -uucore = { workspace = true, features = ["ringbuffer", "lines"] } +thiserror = { workspace = true } +uucore = { workspace = true, features = [ + "parser", + "ringbuffer", + "lines", + "fs", +] } [[bin]] name = "head" diff --git a/src/uu/head/src/head.rs b/src/uu/head/src/head.rs index 3f6fd218507..573926a7bb2 100644 --- a/src/uu/head/src/head.rs +++ b/src/uu/head/src/head.rs @@ -3,25 +3,23 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (vars) BUFWRITER seekable +// spell-checker:ignore (vars) seekable -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; use std::ffi::OsString; -use std::fs::Metadata; -use std::io::{self, BufWriter, ErrorKind, Read, Seek, SeekFrom, Write}; -#[cfg(not(target_os = "windows"))] -use std::os::unix::fs::MetadataExt; +use std::fs::File; +use std::io::{self, BufWriter, Read, Seek, SeekFrom, Write}; +use std::num::TryFromIntError; +#[cfg(unix)] +use std::os::fd::{AsRawFd, FromRawFd}; +use thiserror::Error; use uucore::display::Quotable; -use uucore::error::{FromIo, UResult, USimpleError}; +use uucore::error::{FromIo, UError, UResult}; use uucore::line_ending::LineEnding; -use uucore::lines::lines; use uucore::{format_usage, help_about, help_usage, show}; const BUF_SIZE: usize = 65536; -/// The capacity in bytes for buffered writers. -const BUFWRITER_CAPACITY: usize = 16_384; // 16 kilobytes - const ABOUT: &str = help_about!("head.md"); const USAGE: &str = help_usage!("head.md"); @@ -37,12 +35,43 @@ mod options { mod parse; mod take; -use take::take_all_but; +use take::copy_all_but_n_bytes; +use take::copy_all_but_n_lines; use take::take_lines; +#[derive(Error, Debug)] +enum HeadError { + /// Wrapper around `io::Error` + #[error("error reading {name}: {err}")] + Io { name: String, err: io::Error }, + + #[error("parse error: {0}")] + ParseError(String), + + #[error("bad argument encoding")] + BadEncoding, + + #[error("{0}: number of -bytes or -lines is too large")] + NumTooLarge(#[from] TryFromIntError), + + #[error("clap error: {0}")] + Clap(#[from] clap::Error), + + #[error("{0}")] + MatchOption(String), +} + +impl UError for HeadError { + fn code(&self) -> i32 { + 1 + } +} + +type HeadResult = Result; + pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -155,30 +184,21 @@ impl Mode { fn arg_iterate<'a>( mut args: impl uucore::Args + 'a, -) -> UResult + 'a>> { +) -> HeadResult + 'a>> { // argv[0] is always present let first = args.next().unwrap(); if let Some(second) = args.next() { if let Some(s) = second.to_str() { match parse::parse_obsolete(s) { Some(Ok(iter)) => Ok(Box::new(vec![first].into_iter().chain(iter).chain(args))), - Some(Err(e)) => match e { - parse::ParseError::Syntax => Err(USimpleError::new( - 1, - format!("bad argument format: {}", s.quote()), - )), - parse::ParseError::Overflow => Err(USimpleError::new( - 1, - format!( - "invalid argument: {} Value too large for defined datatype", - s.quote() - ), - )), - }, + Some(Err(parse::ParseError)) => Err(HeadError::ParseError(format!( + "bad argument format: {}", + s.quote() + ))), None => Ok(Box::new(vec![first, second].into_iter().chain(args))), } } else { - Err(USimpleError::new(1, "bad argument encoding".to_owned())) + Err(HeadError::BadEncoding) } } else { Ok(Box::new(vec![first].into_iter())) @@ -197,7 +217,7 @@ struct HeadOptions { impl HeadOptions { ///Construct options from matches - pub fn get_from(matches: &clap::ArgMatches) -> Result { + pub fn get_from(matches: &ArgMatches) -> Result { let mut options = Self::default(); options.quiet = matches.get_flag(options::QUIET_NAME); @@ -211,126 +231,106 @@ impl HeadOptions { Some(v) => v.cloned().collect(), None => vec!["-".to_owned()], }; - //println!("{:#?}", options); + Ok(options) } } -fn read_n_bytes(input: R, n: u64) -> std::io::Result<()> -where - R: Read, -{ +#[inline] +fn wrap_in_stdout_error(err: io::Error) -> io::Error { + io::Error::new( + err.kind(), + format!("error writing 'standard output': {err}"), + ) +} + +fn read_n_bytes(input: impl Read, n: u64) -> io::Result { // Read the first `n` bytes from the `input` reader. let mut reader = input.take(n); // Write those bytes to `stdout`. - let stdout = std::io::stdout(); + let stdout = io::stdout(); let mut stdout = stdout.lock(); - io::copy(&mut reader, &mut stdout)?; + let bytes_written = io::copy(&mut reader, &mut stdout).map_err(wrap_in_stdout_error)?; - Ok(()) + // Make sure we finish writing everything to the target before + // exiting. Otherwise, when Rust is implicitly flushing, any + // error will be silently ignored. + stdout.flush().map_err(wrap_in_stdout_error)?; + + Ok(bytes_written) } -fn read_n_lines(input: &mut impl std::io::BufRead, n: u64, separator: u8) -> std::io::Result<()> { +fn read_n_lines(input: &mut impl io::BufRead, n: u64, separator: u8) -> io::Result { // Read the first `n` lines from the `input` reader. let mut reader = take_lines(input, n, separator); // Write those bytes to `stdout`. - let stdout = std::io::stdout(); + let stdout = io::stdout(); let stdout = stdout.lock(); - let mut writer = BufWriter::with_capacity(BUFWRITER_CAPACITY, stdout); + let mut writer = BufWriter::with_capacity(BUF_SIZE, stdout); - io::copy(&mut reader, &mut writer)?; + let bytes_written = io::copy(&mut reader, &mut writer).map_err(wrap_in_stdout_error)?; - Ok(()) + // Make sure we finish writing everything to the target before + // exiting. Otherwise, when Rust is implicitly flushing, any + // error will be silently ignored. + writer.flush().map_err(wrap_in_stdout_error)?; + + Ok(bytes_written) } fn catch_too_large_numbers_in_backwards_bytes_or_lines(n: u64) -> Option { - match usize::try_from(n) { - Ok(value) => Some(value), - Err(e) => { - show!(USimpleError::new( - 1, - format!("{e}: number of -bytes or -lines is too large") - )); - None - } - } + usize::try_from(n).ok() } -fn read_but_last_n_bytes(input: &mut impl std::io::BufRead, n: u64) -> std::io::Result<()> { - if n == 0 { - //prints everything - return read_n_bytes(input, std::u64::MAX); - } - +fn read_but_last_n_bytes(mut input: impl Read, n: u64) -> io::Result { + let mut bytes_written: u64 = 0; if let Some(n) = catch_too_large_numbers_in_backwards_bytes_or_lines(n) { - let stdout = std::io::stdout(); + let stdout = io::stdout(); let mut stdout = stdout.lock(); - let mut ring_buffer = Vec::new(); - - let mut buffer = [0u8; BUF_SIZE]; - let mut total_read = 0; + bytes_written = copy_all_but_n_bytes(&mut input, &mut stdout, n) + .map_err(wrap_in_stdout_error)? + .try_into() + .unwrap(); - loop { - let read = match input.read(&mut buffer) { - Ok(0) => break, - Ok(read) => read, - Err(e) => match e.kind() { - ErrorKind::Interrupted => continue, - _ => return Err(e), - }, - }; - - total_read += read; - - if total_read <= n { - // Fill the ring buffer without exceeding n bytes - let overflow = total_read - n; - ring_buffer.extend_from_slice(&buffer[..read - overflow]); - } else { - // Write the ring buffer and the part of the buffer that exceeds n - stdout.write_all(&ring_buffer)?; - stdout.write_all(&buffer[..read - n + ring_buffer.len()])?; - ring_buffer.clear(); - ring_buffer.extend_from_slice(&buffer[read - n + ring_buffer.len()..read]); - } - } + // Make sure we finish writing everything to the target before + // exiting. Otherwise, when Rust is implicitly flushing, any + // error will be silently ignored. + stdout.flush().map_err(wrap_in_stdout_error)?; } - - Ok(()) + Ok(bytes_written) } -fn read_but_last_n_lines( - input: impl std::io::BufRead, - n: u64, - separator: u8, -) -> std::io::Result<()> { +fn read_but_last_n_lines(mut input: impl Read, n: u64, separator: u8) -> io::Result { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + if n == 0 { + return io::copy(&mut input, &mut stdout).map_err(wrap_in_stdout_error); + } + let mut bytes_written: u64 = 0; if let Some(n) = catch_too_large_numbers_in_backwards_bytes_or_lines(n) { - let stdout = std::io::stdout(); - let mut stdout = stdout.lock(); - for bytes in take_all_but(lines(input, separator), n) { - stdout.write_all(&bytes?)?; - } + bytes_written = copy_all_but_n_lines(input, &mut stdout, n, separator) + .map_err(wrap_in_stdout_error)? + .try_into() + .unwrap(); + // Make sure we finish writing everything to the target before + // exiting. Otherwise, when Rust is implicitly flushing, any + // error will be silently ignored. + stdout.flush().map_err(wrap_in_stdout_error)?; } - Ok(()) + Ok(bytes_written) } /// Return the index in `input` just after the `n`th line from the end. /// /// If `n` exceeds the number of lines in this file, then return 0. -/// -/// The cursor must be at the start of the seekable input before -/// calling this function. This function rewinds the cursor to the +/// This function rewinds the cursor to the /// beginning of the input just before returning unless there is an /// I/O error. /// -/// If `zeroed` is `false`, interpret the newline character `b'\n'` as -/// a line ending. If `zeroed` is `true`, interpret the null character -/// `b'\0'` as a line ending instead. -/// /// # Errors /// /// This function returns an error if there is a problem seeking @@ -357,24 +357,25 @@ fn read_but_last_n_lines( /// assert_eq!(find_nth_line_from_end(&mut input, 4, false).unwrap(), 0); /// assert_eq!(find_nth_line_from_end(&mut input, 1000, false).unwrap(), 0); /// ``` -fn find_nth_line_from_end(input: &mut R, n: u64, separator: u8) -> std::io::Result +fn find_nth_line_from_end(input: &mut R, n: u64, separator: u8) -> io::Result where R: Read + Seek, { - let size = input.seek(SeekFrom::End(0))?; + let file_size = input.seek(SeekFrom::End(0))?; let mut buffer = [0u8; BUF_SIZE]; - let buf_size: usize = (BUF_SIZE as u64).min(size).try_into().unwrap(); - let buffer = &mut buffer[..buf_size]; let mut i = 0u64; let mut lines = 0u64; loop { // the casts here are ok, `buffer.len()` should never be above a few k - input.seek(SeekFrom::Current( - -((buffer.len() as i64).min((size - i) as i64)), - ))?; + let bytes_remaining_to_search = file_size - i; + let bytes_to_read_this_loop = bytes_remaining_to_search.min(BUF_SIZE.try_into().unwrap()); + let read_start_offset = bytes_remaining_to_search - bytes_to_read_this_loop; + let buffer = &mut buffer[..bytes_to_read_this_loop.try_into().unwrap()]; + + input.seek(SeekFrom::Start(read_start_offset))?; input.read_exact(buffer)?; for byte in buffer.iter().rev() { if byte == &separator { @@ -383,105 +384,66 @@ where // if it were just `n`, if lines == n + 1 { input.rewind()?; - return Ok(size - i); + return Ok(file_size - i); } i += 1; } - if size - i == 0 { + if file_size - i == 0 { input.rewind()?; return Ok(0); } } } -fn is_seekable(input: &mut std::fs::File) -> bool { +fn is_seekable(input: &mut File) -> bool { let current_pos = input.stream_position(); current_pos.is_ok() && input.seek(SeekFrom::End(0)).is_ok() && input.seek(SeekFrom::Start(current_pos.unwrap())).is_ok() } -fn sanity_limited_blksize(_st: &Metadata) -> u64 { - #[cfg(not(target_os = "windows"))] - { - const DEFAULT: u64 = 512; - const MAX: u64 = usize::MAX as u64 / 8 + 1; - - let st_blksize: u64 = _st.blksize(); - match st_blksize { - 0 => DEFAULT, - 1..=MAX => st_blksize, - _ => DEFAULT, - } - } - - #[cfg(target_os = "windows")] - { - 512 - } -} - -fn head_backwards_file(input: &mut std::fs::File, options: &HeadOptions) -> std::io::Result<()> { +fn head_backwards_file(input: &mut File, options: &HeadOptions) -> io::Result { let st = input.metadata()?; let seekable = is_seekable(input); - let blksize_limit = sanity_limited_blksize(&st); + let blksize_limit = uucore::fs::sane_blksize::sane_blksize_from_metadata(&st); if !seekable || st.len() <= blksize_limit { - return head_backwards_without_seek_file(input, options); + head_backwards_without_seek_file(input, options) + } else { + head_backwards_on_seekable_file(input, options) } - - head_backwards_on_seekable_file(input, options) } -fn head_backwards_without_seek_file( - input: &mut std::fs::File, - options: &HeadOptions, -) -> std::io::Result<()> { - let reader = &mut std::io::BufReader::with_capacity(BUF_SIZE, &*input); - +fn head_backwards_without_seek_file(input: &mut File, options: &HeadOptions) -> io::Result { match options.mode { - Mode::AllButLastBytes(n) => read_but_last_n_bytes(reader, n)?, - Mode::AllButLastLines(n) => read_but_last_n_lines(reader, n, options.line_ending.into())?, + Mode::AllButLastBytes(n) => read_but_last_n_bytes(input, n), + Mode::AllButLastLines(n) => read_but_last_n_lines(input, n, options.line_ending.into()), _ => unreachable!(), } - - Ok(()) } -fn head_backwards_on_seekable_file( - input: &mut std::fs::File, - options: &HeadOptions, -) -> std::io::Result<()> { +fn head_backwards_on_seekable_file(input: &mut File, options: &HeadOptions) -> io::Result { match options.mode { Mode::AllButLastBytes(n) => { let size = input.metadata()?.len(); if n >= size { - return Ok(()); + Ok(0) } else { - read_n_bytes( - &mut std::io::BufReader::with_capacity(BUF_SIZE, input), - size - n, - )?; + read_n_bytes(input, size - n) } } Mode::AllButLastLines(n) => { let found = find_nth_line_from_end(input, n, options.line_ending.into())?; - read_n_bytes( - &mut std::io::BufReader::with_capacity(BUF_SIZE, input), - found, - )?; + read_n_bytes(input, found) } _ => unreachable!(), } - Ok(()) } -fn head_file(input: &mut std::fs::File, options: &HeadOptions) -> std::io::Result<()> { +fn head_file(input: &mut File, options: &HeadOptions) -> io::Result { match options.mode { - Mode::FirstBytes(n) => { - read_n_bytes(&mut std::io::BufReader::with_capacity(BUF_SIZE, input), n) - } + Mode::FirstBytes(n) => read_n_bytes(input, n), Mode::FirstLines(n) => read_n_lines( - &mut std::io::BufReader::with_capacity(BUF_SIZE, input), + &mut io::BufReader::with_capacity(BUF_SIZE, input), n, options.line_ending.into(), ), @@ -501,20 +463,45 @@ fn uu_head(options: &HeadOptions) -> UResult<()> { } println!("==> standard input <=="); } - let stdin = std::io::stdin(); - let mut stdin = stdin.lock(); - - match options.mode { - Mode::FirstBytes(n) => read_n_bytes(&mut stdin, n), - Mode::AllButLastBytes(n) => read_but_last_n_bytes(&mut stdin, n), - Mode::FirstLines(n) => read_n_lines(&mut stdin, n, options.line_ending.into()), - Mode::AllButLastLines(n) => { - read_but_last_n_lines(&mut stdin, n, options.line_ending.into()) + let stdin = io::stdin(); + + #[cfg(unix)] + { + let stdin_raw_fd = stdin.as_raw_fd(); + let mut stdin_file = unsafe { File::from_raw_fd(stdin_raw_fd) }; + let current_pos = stdin_file.stream_position(); + if let Ok(current_pos) = current_pos { + // We have a seekable file. Ensure we set the input stream to the + // last byte read so that any tools that parse the remainder of + // the stdin stream read from the correct place. + + let bytes_read = head_file(&mut stdin_file, options)?; + stdin_file.seek(SeekFrom::Start(current_pos + bytes_read))?; + } else { + let _bytes_read = head_file(&mut stdin_file, options)?; } } + + #[cfg(not(unix))] + { + let mut stdin = stdin.lock(); + + match options.mode { + Mode::FirstBytes(n) => read_n_bytes(&mut stdin, n), + Mode::AllButLastBytes(n) => read_but_last_n_bytes(&mut stdin, n), + Mode::FirstLines(n) => { + read_n_lines(&mut stdin, n, options.line_ending.into()) + } + Mode::AllButLastLines(n) => { + read_but_last_n_lines(&mut stdin, n, options.line_ending.into()) + } + }?; + } + + Ok(()) } (name, false) => { - let mut file = match std::fs::File::open(name) { + let mut file = match File::open(name) { Ok(f) => f, Err(err) => { show!(err.map_err_context(|| format!( @@ -530,19 +517,21 @@ fn uu_head(options: &HeadOptions) -> UResult<()> { } println!("==> {name} <=="); } - head_file(&mut file, options) + head_file(&mut file, options)?; + Ok(()) } }; - if res.is_err() { + if let Err(e) = res { let name = if file.as_str() == "-" { "standard input" } else { file }; - show!(USimpleError::new( - 1, - format!("error reading {name}: Input/output error") - )); + return Err(HeadError::Io { + name: name.to_string(), + err: e, + } + .into()); } first = false; } @@ -559,7 +548,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let args = match HeadOptions::get_from(&matches) { Ok(o) => o, Err(s) => { - return Err(USimpleError::new(1, s)); + return Err(HeadError::MatchOption(s).into()); } }; uu_head(&args) @@ -567,8 +556,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { #[cfg(test)] mod tests { + use io::Cursor; use std::ffi::OsString; - use std::io::Cursor; use super::*; @@ -591,7 +580,7 @@ mod tests { #[test] fn test_gnu_compatibility() { let args = options("-n 1 -c 1 -n 5 -c kiB -vqvqv").unwrap(); // spell-checker:disable-line - assert!(args.mode == Mode::FirstBytes(1024)); + assert_eq!(args.mode, Mode::FirstBytes(1024)); assert!(args.verbose); assert_eq!(options("-5").unwrap().mode, Mode::FirstLines(5)); assert_eq!(options("-2b").unwrap().mode, Mode::FirstBytes(1024)); @@ -667,7 +656,7 @@ mod tests { //test that bad obsoletes are an error assert!(arg_outputs("head -123FooBar").is_err()); //test overflow - assert!(arg_outputs("head -100000000000000000000000000000000000000000").is_err()); + assert!(arg_outputs("head -100000000000000000000000000000000000000000").is_ok()); //test that empty args remain unchanged assert_eq!(arg_outputs("head"), Ok("head".to_owned())); } @@ -683,19 +672,66 @@ mod tests { #[test] fn read_early_exit() { - let mut empty = std::io::BufReader::new(std::io::Cursor::new(Vec::new())); + let mut empty = io::BufReader::new(Cursor::new(Vec::new())); assert!(read_n_bytes(&mut empty, 0).is_ok()); assert!(read_n_lines(&mut empty, 0, b'\n').is_ok()); } #[test] fn test_find_nth_line_from_end() { - let mut input = Cursor::new("x\ny\nz\n"); - assert_eq!(find_nth_line_from_end(&mut input, 0, b'\n').unwrap(), 6); - assert_eq!(find_nth_line_from_end(&mut input, 1, b'\n').unwrap(), 4); - assert_eq!(find_nth_line_from_end(&mut input, 2, b'\n').unwrap(), 2); - assert_eq!(find_nth_line_from_end(&mut input, 3, b'\n').unwrap(), 0); - assert_eq!(find_nth_line_from_end(&mut input, 4, b'\n').unwrap(), 0); - assert_eq!(find_nth_line_from_end(&mut input, 1000, b'\n').unwrap(), 0); + // Make sure our input buffer is several multiples of BUF_SIZE in size + // such that we can be reasonably confident we've exercised all logic paths. + // Make the contents of the buffer look like... + // aaaa\n + // aaaa\n + // aaaa\n + // aaaa\n + // aaaa\n + // ... + // This will make it easier to validate the results since each line will have + // 5 bytes in it. + + let minimum_buffer_size = BUF_SIZE * 4; + let mut input_buffer = vec![]; + let mut loop_iteration: u64 = 0; + while input_buffer.len() < minimum_buffer_size { + for _n in 0..4 { + input_buffer.push(b'a'); + } + loop_iteration += 1; + input_buffer.push(b'\n'); + } + + let lines_in_input_file = loop_iteration; + let input_length = lines_in_input_file * 5; + assert_eq!(input_length, input_buffer.len().try_into().unwrap()); + let mut input = Cursor::new(input_buffer); + // We now have loop_iteration lines in the buffer Now walk backwards through the buffer + // to confirm everything parses correctly. + // Use a large step size to prevent the test from taking too long, but don't use a power + // of 2 in case we miss some corner case. + let step_size = 511; + for n in (0..lines_in_input_file).filter(|v| v % step_size == 0) { + // The 5*n comes from 5-bytes per row. + assert_eq!( + find_nth_line_from_end(&mut input, n, b'\n').unwrap(), + input_length - 5 * n + ); + } + + // Now confirm that if we query with a value >= lines_in_input_file we get an offset + // of 0 + assert_eq!( + find_nth_line_from_end(&mut input, lines_in_input_file, b'\n').unwrap(), + 0 + ); + assert_eq!( + find_nth_line_from_end(&mut input, lines_in_input_file + 1, b'\n').unwrap(), + 0 + ); + assert_eq!( + find_nth_line_from_end(&mut input, lines_in_input_file + 1000, b'\n').unwrap(), + 0 + ); } } diff --git a/src/uu/head/src/parse.rs b/src/uu/head/src/parse.rs index dce60bae012..a4ce6e71069 100644 --- a/src/uu/head/src/parse.rs +++ b/src/uu/head/src/parse.rs @@ -4,33 +4,37 @@ // file that was distributed with this source code. use std::ffi::OsString; -use uucore::parse_size::{parse_size_u64, ParseSizeError}; +use uucore::parser::parse_size::{ParseSizeError, parse_size_u64_max}; #[derive(PartialEq, Eq, Debug)] -pub enum ParseError { - Syntax, - Overflow, -} +pub struct ParseError; /// Parses obsolete syntax /// head -NUM\[kmzv\] // spell-checker:disable-line -pub fn parse_obsolete(src: &str) -> Option, ParseError>> { +pub fn parse_obsolete(src: &str) -> Option, ParseError>> { let mut chars = src.char_indices(); - if let Some((_, '-')) = chars.next() { - let mut num_end = 0usize; + if let Some((mut num_start, '-')) = chars.next() { + num_start += 1; + let mut num_end = src.len(); let mut has_num = false; + let mut plus_possible = false; let mut last_char = 0 as char; for (n, c) in &mut chars { if c.is_ascii_digit() { has_num = true; - num_end = n; + plus_possible = false; + } else if c == '+' && plus_possible { + plus_possible = false; + num_start += 1; + continue; } else { + num_end = n; last_char = c; break; } } if has_num { - process_num_block(&src[1..=num_end], last_char, &mut chars) + process_num_block(&src[num_start..num_end], last_char, &mut chars) } else { None } @@ -44,66 +48,63 @@ fn process_num_block( src: &str, last_char: char, chars: &mut std::str::CharIndices, -) -> Option, ParseError>> { - match src.parse::() { - Ok(num) => { - let mut quiet = false; - let mut verbose = false; - let mut zero_terminated = false; - let mut multiplier = None; - let mut c = last_char; - loop { - // note that here, we only match lower case 'k', 'c', and 'm' - match c { - // we want to preserve order - // this also saves us 1 heap allocation - 'q' => { - quiet = true; - verbose = false; - } - 'v' => { - verbose = true; - quiet = false; - } - 'z' => zero_terminated = true, - 'c' => multiplier = Some(1), - 'b' => multiplier = Some(512), - 'k' => multiplier = Some(1024), - 'm' => multiplier = Some(1024 * 1024), - '\0' => {} - _ => return Some(Err(ParseError::Syntax)), - } - if let Some((_, next)) = chars.next() { - c = next; - } else { - break; - } - } - let mut options = Vec::new(); - if quiet { - options.push(OsString::from("-q")); - } - if verbose { - options.push(OsString::from("-v")); +) -> Option, ParseError>> { + let num = match src.parse::() { + Ok(n) => n, + Err(e) if *e.kind() == std::num::IntErrorKind::PosOverflow => usize::MAX, + _ => return Some(Err(ParseError)), + }; + let mut quiet = false; + let mut verbose = false; + let mut zero_terminated = false; + let mut multiplier = None; + let mut c = last_char; + loop { + // note that here, we only match lower case 'k', 'c', and 'm' + match c { + // we want to preserve order + // this also saves us 1 heap allocation + 'q' => { + quiet = true; + verbose = false; } - if zero_terminated { - options.push(OsString::from("-z")); + 'v' => { + verbose = true; + quiet = false; } - if let Some(n) = multiplier { - options.push(OsString::from("-c")); - let num = match num.checked_mul(n) { - Some(n) => n, - None => return Some(Err(ParseError::Overflow)), - }; - options.push(OsString::from(format!("{num}"))); - } else { - options.push(OsString::from("-n")); - options.push(OsString::from(format!("{num}"))); - } - Some(Ok(options.into_iter())) + 'z' => zero_terminated = true, + 'c' => multiplier = Some(1), + 'b' => multiplier = Some(512), + 'k' => multiplier = Some(1024), + 'm' => multiplier = Some(1024 * 1024), + '\0' => {} + _ => return Some(Err(ParseError)), + } + if let Some((_, next)) = chars.next() { + c = next; + } else { + break; } - Err(_) => Some(Err(ParseError::Overflow)), } + let mut options = Vec::new(); + if quiet { + options.push(OsString::from("-q")); + } + if verbose { + options.push(OsString::from("-v")); + } + if zero_terminated { + options.push(OsString::from("-z")); + } + if let Some(n) = multiplier { + options.push(OsString::from("-c")); + let num = num.saturating_mul(n); + options.push(OsString::from(format!("{num}"))); + } else { + options.push(OsString::from("-n")); + options.push(OsString::from(format!("{num}"))); + } + Some(Ok(options)) } /// Parses an -c or -n argument, @@ -129,7 +130,7 @@ pub fn parse_num(src: &str) -> Result<(u64, bool), ParseSizeError> { if trimmed_string.is_empty() { Ok((0, all_but_last)) } else { - parse_size_u64(trimmed_string).map(|n| (n, all_but_last)) + parse_size_u64_max(trimmed_string).map(|n| (n, all_but_last)) } } @@ -141,7 +142,10 @@ mod tests { let r = parse_obsolete(src); match r { Some(s) => match s { - Ok(v) => Some(Ok(v.map(|s| s.to_str().unwrap().to_owned()).collect())), + Ok(v) => Some(Ok(v + .into_iter() + .map(|s| s.to_str().unwrap().to_owned()) + .collect())), Err(e) => Some(Err(e)), }, None => None, @@ -175,8 +179,8 @@ mod tests { #[test] fn test_parse_errors_obsolete() { - assert_eq!(obsolete("-5n"), Some(Err(ParseError::Syntax))); - assert_eq!(obsolete("-5c5"), Some(Err(ParseError::Syntax))); + assert_eq!(obsolete("-5n"), Some(Err(ParseError))); + assert_eq!(obsolete("-5c5"), Some(Err(ParseError))); } #[test] @@ -190,18 +194,24 @@ mod tests { fn test_parse_obsolete_overflow_x64() { assert_eq!( obsolete("-1000000000000000m"), - Some(Err(ParseError::Overflow)) + obsolete_result(&["-c", "18446744073709551615"]) ); assert_eq!( obsolete("-10000000000000000000000"), - Some(Err(ParseError::Overflow)) + obsolete_result(&["-n", "18446744073709551615"]) ); } #[test] #[cfg(target_pointer_width = "32")] fn test_parse_obsolete_overflow_x32() { - assert_eq!(obsolete("-42949672960"), Some(Err(ParseError::Overflow))); - assert_eq!(obsolete("-42949672k"), Some(Err(ParseError::Overflow))); + assert_eq!( + obsolete("-42949672960"), + obsolete_result(&["-n", "4294967295"]) + ); + assert_eq!( + obsolete("-42949672k"), + obsolete_result(&["-c", "4294967295"]) + ); } } diff --git a/src/uu/head/src/take.rs b/src/uu/head/src/take.rs index da48afd6a86..1a303bbd401 100644 --- a/src/uu/head/src/take.rs +++ b/src/uu/head/src/take.rs @@ -3,67 +3,308 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. //! Take all but the last elements of an iterator. -use std::io::Read; - use memchr::memchr_iter; +use std::collections::VecDeque; +use std::io::{ErrorKind, Read, Write}; -use uucore::ringbuffer::RingBuffer; +const BUF_SIZE: usize = 65536; -/// Create an iterator over all but the last `n` elements of `iter`. -/// -/// # Examples -/// -/// ```rust,ignore -/// let data = [1, 2, 3, 4, 5]; -/// let n = 2; -/// let mut iter = take_all_but(data.iter(), n); -/// assert_eq!(Some(4), iter.next()); -/// assert_eq!(Some(5), iter.next()); -/// assert_eq!(None, iter.next()); -/// ``` -pub fn take_all_but(iter: I, n: usize) -> TakeAllBut { - TakeAllBut::new(iter, n) +struct TakeAllBuffer { + buffer: Vec, + start_index: usize, } -/// An iterator that only iterates over the last elements of another iterator. -pub struct TakeAllBut { - iter: I, - buf: RingBuffer<::Item>, +impl TakeAllBuffer { + fn new() -> Self { + TakeAllBuffer { + buffer: vec![], + start_index: 0, + } + } + + fn fill_buffer(&mut self, reader: &mut impl Read) -> std::io::Result { + self.buffer.resize(BUF_SIZE, 0); + self.start_index = 0; + loop { + match reader.read(&mut self.buffer[..]) { + Ok(n) => { + self.buffer.truncate(n); + return Ok(n); + } + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + } + } + + fn write_bytes_exact(&mut self, writer: &mut impl Write, bytes: usize) -> std::io::Result<()> { + let buffer_to_write = &self.remaining_buffer()[..bytes]; + writer.write_all(buffer_to_write)?; + self.start_index += bytes; + assert!(self.start_index <= self.buffer.len()); + Ok(()) + } + + fn write_all(&mut self, writer: &mut impl Write) -> std::io::Result { + let remaining_bytes = self.remaining_bytes(); + self.write_bytes_exact(writer, remaining_bytes)?; + Ok(remaining_bytes) + } + + fn write_bytes_limit( + &mut self, + writer: &mut impl Write, + max_bytes: usize, + ) -> std::io::Result { + let bytes_to_write = self.remaining_bytes().min(max_bytes); + self.write_bytes_exact(writer, bytes_to_write)?; + Ok(bytes_to_write) + } + + fn remaining_buffer(&self) -> &[u8] { + &self.buffer[self.start_index..] + } + + fn remaining_bytes(&self) -> usize { + self.remaining_buffer().len() + } + + fn is_empty(&self) -> bool { + assert!(self.start_index <= self.buffer.len()); + self.start_index == self.buffer.len() + } } -impl TakeAllBut { - pub fn new(mut iter: I, n: usize) -> Self { - // Create a new ring buffer and fill it up. - // - // If there are fewer than `n` elements in `iter`, then we - // exhaust the iterator so that whenever `TakeAllBut::next()` is - // called, it will return `None`, as expected. - let mut buf = RingBuffer::new(n); - for _ in 0..n { - let value = match iter.next() { - None => { +/// Function to copy all but `n` bytes from the reader to the writer. +/// +/// If `n` exceeds the number of bytes in the input file then nothing is copied. +/// If no errors are encountered then the function returns the number of bytes +/// copied. +/// +/// Algorithm for this function is as follows... +/// 1 - Chunks of the input file are read into a queue of TakeAllBuffer instances. +/// Chunks are read until at least we have enough data to write out the entire contents of the +/// first TakeAllBuffer in the queue whilst still retaining at least `n` bytes in the queue. +/// If we hit EoF at any point, stop reading. +/// 2 - Assess whether we managed to queue up greater-than `n` bytes. If not, we must be done, in +/// which case break and return. +/// 3 - Write either the full first buffer of data, or just enough bytes to get back down to having +/// the required `n` bytes of data queued. +/// 4 - Go back to (1). +pub fn copy_all_but_n_bytes( + reader: &mut impl Read, + writer: &mut impl Write, + n: usize, +) -> std::io::Result { + let mut buffers: VecDeque = VecDeque::new(); + let mut empty_buffer_pool: Vec = vec![]; + let mut buffered_bytes: usize = 0; + let mut total_bytes_copied = 0; + loop { + loop { + // Try to buffer at least enough to write the entire first buffer. + let front_buffer = buffers.front(); + if let Some(front_buffer) = front_buffer { + if buffered_bytes >= n + front_buffer.remaining_bytes() { break; } - Some(x) => x, + } + let mut new_buffer = empty_buffer_pool.pop().unwrap_or_else(TakeAllBuffer::new); + let filled_bytes = new_buffer.fill_buffer(reader)?; + if filled_bytes == 0 { + // filled_bytes==0 => Eof + break; + } + buffers.push_back(new_buffer); + buffered_bytes += filled_bytes; + } + + // If we've got <=n bytes buffered here we have nothing left to do. + if buffered_bytes <= n { + break; + } + + let excess_buffered_bytes = buffered_bytes - n; + // Since we have some data buffered, can assume we have >=1 buffer - i.e. safe to unwrap. + let front_buffer = buffers.front_mut().unwrap(); + let bytes_written = front_buffer.write_bytes_limit(writer, excess_buffered_bytes)?; + buffered_bytes -= bytes_written; + total_bytes_copied += bytes_written; + // If the front buffer is empty (which it probably is), push it into the empty-buffer-pool. + if front_buffer.is_empty() { + empty_buffer_pool.push(buffers.pop_front().unwrap()); + } + } + Ok(total_bytes_copied) +} + +struct TakeAllLinesBuffer { + inner: TakeAllBuffer, + terminated_lines: usize, + partial_line: bool, +} + +struct BytesAndLines { + bytes: usize, + terminated_lines: usize, +} + +impl TakeAllLinesBuffer { + fn new() -> Self { + TakeAllLinesBuffer { + inner: TakeAllBuffer::new(), + terminated_lines: 0, + partial_line: false, + } + } + + fn fill_buffer( + &mut self, + reader: &mut impl Read, + separator: u8, + ) -> std::io::Result { + let bytes_read = self.inner.fill_buffer(reader)?; + // Count the number of lines... + self.terminated_lines = memchr_iter(separator, self.inner.remaining_buffer()).count(); + if let Some(last_char) = self.inner.remaining_buffer().last() { + if *last_char != separator { + self.partial_line = true; + } + } + Ok(BytesAndLines { + bytes: bytes_read, + terminated_lines: self.terminated_lines, + }) + } + + fn write_lines( + &mut self, + writer: &mut impl Write, + max_lines: usize, + separator: u8, + ) -> std::io::Result { + assert!(max_lines > 0, "Must request at least 1 line."); + let ret; + if max_lines > self.terminated_lines { + ret = BytesAndLines { + bytes: self.inner.write_all(writer)?, + terminated_lines: self.terminated_lines, + }; + self.terminated_lines = 0; + } else { + let index = memchr_iter(separator, self.inner.remaining_buffer()).nth(max_lines - 1); + assert!( + index.is_some(), + "Somehow we're being asked to write more lines than we have, that's a bug in copy_all_but_lines." + ); + let index = index.unwrap(); + // index is the offset of the separator character, zero indexed. Need to add 1 to get the number + // of bytes to write. + let bytes_to_write = index + 1; + self.inner.write_bytes_exact(writer, bytes_to_write)?; + ret = BytesAndLines { + bytes: bytes_to_write, + terminated_lines: max_lines, }; - buf.push_back(value); + self.terminated_lines -= max_lines; } - Self { iter, buf } + Ok(ret) + } + + fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + fn terminated_lines(&self) -> usize { + self.terminated_lines + } + + fn partial_line(&self) -> bool { + self.partial_line } } -impl Iterator for TakeAllBut -where - I: Iterator, -{ - type Item = ::Item; +/// Function to copy all but `n` lines from the reader to the writer. +/// +/// Lines are inferred from the `separator` value passed in by the client. +/// If `n` exceeds the number of lines in the input file then nothing is copied. +/// The last line in the file is not required to end with a `separator` character. +/// If no errors are encountered then they function returns the number of bytes +/// copied. +/// +/// Algorithm for this function is as follows... +/// 1 - Chunks of the input file are read into a queue of TakeAllLinesBuffer instances. +/// Chunks are read until at least we have enough lines that we can write out the entire +/// contents of the first TakeAllLinesBuffer in the queue whilst still retaining at least +/// `n` lines in the queue. +/// If we hit EoF at any point, stop reading. +/// 2 - Asses whether we managed to queue up greater-than `n` lines. If not, we must be done, in +/// which case break and return. +/// 3 - Write either the full first buffer of data, or just enough lines to get back down to +/// having the required `n` lines of data queued. +/// 4 - Go back to (1). +/// +/// Note that lines will regularly straddle multiple TakeAllLinesBuffer instances. The partial_line +/// flag on TakeAllLinesBuffer tracks this, and we use that to ensure that we write out enough +/// lines in the case that the input file doesn't end with a `separator` character. +pub fn copy_all_but_n_lines( + mut reader: R, + writer: &mut W, + n: usize, + separator: u8, +) -> std::io::Result { + // This function requires `n` > 0. Assert it! + assert!(n > 0); + let mut buffers: VecDeque = VecDeque::new(); + let mut buffered_terminated_lines: usize = 0; + let mut empty_buffers = vec![]; + let mut total_bytes_copied = 0; + loop { + // Try to buffer enough such that we can write out the entire first buffer. + loop { + // First check if we have enough lines buffered that we can write out the entire + // front buffer. If so, break. + let front_buffer = buffers.front(); + if let Some(front_buffer) = front_buffer { + if buffered_terminated_lines > n + front_buffer.terminated_lines() { + break; + } + } + // Else we need to try to buffer more data... + let mut new_buffer = empty_buffers.pop().unwrap_or_else(TakeAllLinesBuffer::new); + let fill_result = new_buffer.fill_buffer(&mut reader, separator)?; + if fill_result.bytes == 0 { + // fill_result.bytes == 0 => EoF. + break; + } + buffered_terminated_lines += fill_result.terminated_lines; + buffers.push_back(new_buffer); + } - fn next(&mut self) -> Option<::Item> { - match self.iter.next() { - Some(value) => self.buf.push_back(value), - None => None, + // If we've not buffered more lines than we need to hold back we must be done. + if buffered_terminated_lines < n + || (buffered_terminated_lines == n && !buffers.back().unwrap().partial_line()) + { + break; + } + + let excess_buffered_terminated_lines = buffered_terminated_lines - n; + // Since we have some data buffered can assume we have at least 1 buffer, so safe to unwrap. + let lines_to_write = if buffers.back().unwrap().partial_line() { + excess_buffered_terminated_lines + 1 + } else { + excess_buffered_terminated_lines + }; + let front_buffer = buffers.front_mut().unwrap(); + let write_result = front_buffer.write_lines(writer, lines_to_write, separator)?; + buffered_terminated_lines -= write_result.terminated_lines; + total_bytes_copied += write_result.bytes; + // If the front buffer is empty (which it probably is), push it into the empty-buffer-pool. + if front_buffer.is_empty() { + empty_buffers.push(buffers.pop_front().unwrap()); } } + Ok(total_bytes_copied) } /// Like `std::io::Take`, but for lines instead of bytes. @@ -118,38 +359,285 @@ pub fn take_lines(reader: R, limit: u64, separator: u8) -> TakeLines { #[cfg(test)] mod tests { - use std::io::BufRead; - use std::io::BufReader; + use std::io::{BufRead, BufReader}; - use crate::take::take_all_but; - use crate::take::take_lines; + use crate::take::{ + TakeAllBuffer, TakeAllLinesBuffer, copy_all_but_n_bytes, copy_all_but_n_lines, take_lines, + }; #[test] - fn test_fewer_elements() { - let mut iter = take_all_but([0, 1, 2].iter(), 2); - assert_eq!(Some(&0), iter.next()); - assert_eq!(None, iter.next()); + fn test_take_all_buffer_exact_bytes() { + let input_buffer = "abc"; + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut take_all_buffer = TakeAllBuffer::new(); + let bytes_read = take_all_buffer.fill_buffer(&mut input_reader).unwrap(); + assert_eq!(bytes_read, input_buffer.len()); + assert_eq!(take_all_buffer.remaining_bytes(), input_buffer.len()); + assert_eq!(take_all_buffer.remaining_buffer(), input_buffer.as_bytes()); + assert!(!take_all_buffer.is_empty()); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + for (index, c) in input_buffer.bytes().enumerate() { + take_all_buffer + .write_bytes_exact(&mut output_reader, 1) + .unwrap(); + let buf_ref = output_reader.get_ref(); + assert_eq!(buf_ref.len(), index + 1); + assert_eq!(buf_ref[index], c); + assert_eq!( + take_all_buffer.remaining_bytes(), + input_buffer.len() - (index + 1) + ); + assert_eq!( + take_all_buffer.remaining_buffer(), + &input_buffer.as_bytes()[index + 1..] + ); + } + + assert!(take_all_buffer.is_empty()); + assert_eq!(take_all_buffer.remaining_bytes(), 0); + assert_eq!(take_all_buffer.remaining_buffer(), "".as_bytes()); } #[test] - fn test_same_number_of_elements() { - let mut iter = take_all_but([0, 1].iter(), 2); - assert_eq!(None, iter.next()); + fn test_take_all_buffer_all_bytes() { + let input_buffer = "abc"; + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut take_all_buffer = TakeAllBuffer::new(); + let bytes_read = take_all_buffer.fill_buffer(&mut input_reader).unwrap(); + assert_eq!(bytes_read, input_buffer.len()); + assert_eq!(take_all_buffer.remaining_bytes(), input_buffer.len()); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_written = take_all_buffer.write_all(&mut output_reader).unwrap(); + assert_eq!(bytes_written, input_buffer.len()); + assert_eq!(output_reader.get_ref().as_slice(), input_buffer.as_bytes()); + + assert!(take_all_buffer.is_empty()); + assert_eq!(take_all_buffer.remaining_bytes(), 0); + assert_eq!(take_all_buffer.remaining_buffer(), "".as_bytes()); + + // Now do a write_all on an empty TakeAllBuffer. Confirm correct behavior. + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_written = take_all_buffer.write_all(&mut output_reader).unwrap(); + assert_eq!(bytes_written, 0); + assert_eq!(output_reader.get_ref().as_slice().len(), 0); } #[test] - fn test_more_elements() { - let mut iter = take_all_but([0].iter(), 2); - assert_eq!(None, iter.next()); + fn test_take_all_buffer_limit_bytes() { + let input_buffer = "abc"; + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut take_all_buffer = TakeAllBuffer::new(); + let bytes_read = take_all_buffer.fill_buffer(&mut input_reader).unwrap(); + assert_eq!(bytes_read, input_buffer.len()); + assert_eq!(take_all_buffer.remaining_bytes(), input_buffer.len()); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + // Write all but 1 bytes. + let bytes_to_write = input_buffer.len() - 1; + let bytes_written = take_all_buffer + .write_bytes_limit(&mut output_reader, bytes_to_write) + .unwrap(); + assert_eq!(bytes_written, bytes_to_write); + assert_eq!( + output_reader.get_ref().as_slice(), + &input_buffer.as_bytes()[..bytes_to_write] + ); + assert!(!take_all_buffer.is_empty()); + assert_eq!(take_all_buffer.remaining_bytes(), 1); + assert_eq!( + take_all_buffer.remaining_buffer(), + &input_buffer.as_bytes()[bytes_to_write..] + ); + + // Write 1 more byte - i.e. last byte in buffer. + let bytes_to_write = 1; + let bytes_written = take_all_buffer + .write_bytes_limit(&mut output_reader, bytes_to_write) + .unwrap(); + assert_eq!(bytes_written, bytes_to_write); + assert_eq!(output_reader.get_ref().as_slice(), input_buffer.as_bytes()); + assert!(take_all_buffer.is_empty()); + assert_eq!(take_all_buffer.remaining_bytes(), 0); + assert_eq!(take_all_buffer.remaining_buffer(), "".as_bytes()); + + // Write 1 more byte - i.e. confirm behavior on already empty buffer. + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_to_write = 1; + let bytes_written = take_all_buffer + .write_bytes_limit(&mut output_reader, bytes_to_write) + .unwrap(); + assert_eq!(bytes_written, 0); + assert_eq!(output_reader.get_ref().as_slice().len(), 0); + assert!(take_all_buffer.is_empty()); + assert_eq!(take_all_buffer.remaining_bytes(), 0); + assert_eq!(take_all_buffer.remaining_buffer(), "".as_bytes()); } #[test] - fn test_zero_elements() { - let mut iter = take_all_but([0, 1, 2].iter(), 0); - assert_eq!(Some(&0), iter.next()); - assert_eq!(Some(&1), iter.next()); - assert_eq!(Some(&2), iter.next()); - assert_eq!(None, iter.next()); + #[allow(clippy::cognitive_complexity)] + fn test_take_all_lines_buffer() { + // 3 lines with new-lines and one partial line. + let input_buffer = "a\nb\nc\ndef"; + let separator = b'\n'; + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut take_all_lines_buffer = TakeAllLinesBuffer::new(); + let fill_result = take_all_lines_buffer + .fill_buffer(&mut input_reader, separator) + .unwrap(); + assert_eq!(fill_result.bytes, input_buffer.len()); + assert_eq!(fill_result.terminated_lines, 3); + assert_eq!(take_all_lines_buffer.terminated_lines(), 3); + assert!(!take_all_lines_buffer.is_empty()); + assert!(take_all_lines_buffer.partial_line()); + + // Write 1st line. + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let lines_to_write = 1; + let write_result = take_all_lines_buffer + .write_lines(&mut output_reader, lines_to_write, separator) + .unwrap(); + assert_eq!(write_result.bytes, 2); + assert_eq!(write_result.terminated_lines, lines_to_write); + assert_eq!(output_reader.get_ref().as_slice(), "a\n".as_bytes()); + assert!(!take_all_lines_buffer.is_empty()); + assert_eq!(take_all_lines_buffer.terminated_lines(), 2); + + // Write 2nd line. + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let lines_to_write = 1; + let write_result = take_all_lines_buffer + .write_lines(&mut output_reader, lines_to_write, separator) + .unwrap(); + assert_eq!(write_result.bytes, 2); + assert_eq!(write_result.terminated_lines, lines_to_write); + assert_eq!(output_reader.get_ref().as_slice(), "b\n".as_bytes()); + assert!(!take_all_lines_buffer.is_empty()); + assert_eq!(take_all_lines_buffer.terminated_lines(), 1); + + // Now try to write 3 lines even though we have only 1 line remaining. Should write everything left in the buffer. + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let lines_to_write = 3; + let write_result = take_all_lines_buffer + .write_lines(&mut output_reader, lines_to_write, separator) + .unwrap(); + assert_eq!(write_result.bytes, 5); + assert_eq!(write_result.terminated_lines, 1); + assert_eq!(output_reader.get_ref().as_slice(), "c\ndef".as_bytes()); + assert!(take_all_lines_buffer.is_empty()); + assert_eq!(take_all_lines_buffer.terminated_lines(), 0); + + // Test empty buffer. + let input_buffer = ""; + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut take_all_lines_buffer = TakeAllLinesBuffer::new(); + let fill_result = take_all_lines_buffer + .fill_buffer(&mut input_reader, separator) + .unwrap(); + assert_eq!(fill_result.bytes, 0); + assert_eq!(fill_result.terminated_lines, 0); + assert_eq!(take_all_lines_buffer.terminated_lines(), 0); + assert!(take_all_lines_buffer.is_empty()); + assert!(!take_all_lines_buffer.partial_line()); + + // Test buffer that ends with newline. + let input_buffer = "\n"; + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut take_all_lines_buffer = TakeAllLinesBuffer::new(); + let fill_result = take_all_lines_buffer + .fill_buffer(&mut input_reader, separator) + .unwrap(); + assert_eq!(fill_result.bytes, 1); + assert_eq!(fill_result.terminated_lines, 1); + assert_eq!(take_all_lines_buffer.terminated_lines(), 1); + assert!(!take_all_lines_buffer.is_empty()); + assert!(!take_all_lines_buffer.partial_line()); + } + + #[test] + fn test_copy_all_but_n_bytes() { + // Test the copy_all_but_bytes fn. Test several scenarios... + // 1 - Hold back more bytes than the input will provide. Should have nothing written to output. + let input_buffer = "a\nb\nc\ndef"; + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_copied = copy_all_but_n_bytes( + &mut input_reader, + &mut output_reader, + input_buffer.len() + 1, + ) + .unwrap(); + assert_eq!(bytes_copied, 0); + + // 2 - Hold back exactly the number of bytes the input will provide. Should have nothing written to output. + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_copied = + copy_all_but_n_bytes(&mut input_reader, &mut output_reader, input_buffer.len()) + .unwrap(); + assert_eq!(bytes_copied, 0); + + // 3 - Hold back 1 fewer byte than input will provide. Should have one byte written to output. + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_copied = copy_all_but_n_bytes( + &mut input_reader, + &mut output_reader, + input_buffer.len() - 1, + ) + .unwrap(); + assert_eq!(bytes_copied, 1); + assert_eq!(output_reader.get_ref()[..], input_buffer.as_bytes()[0..1]); + } + + #[test] + fn test_copy_all_but_n_lines() { + // Test the copy_all_but_lines fn. Test several scenarios... + // 1 - Hold back more lines than the input will provide. Should have nothing written to output. + let input_buffer = "a\nb\nc\ndef"; + let separator = b'\n'; + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_copied = + copy_all_but_n_lines(&mut input_reader, &mut output_reader, 5, separator).unwrap(); + assert_eq!(bytes_copied, 0); + + // 2 - Hold back exactly the number of lines the input will provide. Should have nothing written to output. + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_copied = + copy_all_but_n_lines(&mut input_reader, &mut output_reader, 4, separator).unwrap(); + assert_eq!(bytes_copied, 0); + + // 3 - Hold back 1 fewer lines than input will provide. Should have one line written to output. + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_copied = + copy_all_but_n_lines(&mut input_reader, &mut output_reader, 3, separator).unwrap(); + assert_eq!(bytes_copied, 2); + assert_eq!(output_reader.get_ref()[..], input_buffer.as_bytes()[0..2]); + + // Now test again with an input that has a new-line ending... + // 4 - Hold back more lines than the input will provide. Should have nothing written to output. + let input_buffer = "a\nb\nc\ndef\n"; + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_copied = + copy_all_but_n_lines(&mut input_reader, &mut output_reader, 5, separator).unwrap(); + assert_eq!(bytes_copied, 0); + + // 5 - Hold back exactly the number of lines the input will provide. Should have nothing written to output. + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_copied = + copy_all_but_n_lines(&mut input_reader, &mut output_reader, 4, separator).unwrap(); + assert_eq!(bytes_copied, 0); + + // 6 - Hold back 1 fewer lines than input will provide. Should have one line written to output. + let mut input_reader = std::io::Cursor::new(input_buffer); + let mut output_reader = std::io::Cursor::new(vec![0x10; 0]); + let bytes_copied = + copy_all_but_n_lines(&mut input_reader, &mut output_reader, 3, separator).unwrap(); + assert_eq!(bytes_copied, 2); + assert_eq!(output_reader.get_ref()[..], input_buffer.as_bytes()[0..2]); } #[test] diff --git a/src/uu/hostid/Cargo.toml b/src/uu/hostid/Cargo.toml index 8e551befb4a..3599f0847bf 100644 --- a/src/uu/hostid/Cargo.toml +++ b/src/uu/hostid/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_hostid" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "hostid ~ (uutils) display the numeric identifier of the current host" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/hostid" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/hostid.rs" diff --git a/src/uu/hostid/src/hostid.rs b/src/uu/hostid/src/hostid.rs index a5c18d07563..a01151dde17 100644 --- a/src/uu/hostid/src/hostid.rs +++ b/src/uu/hostid/src/hostid.rs @@ -5,28 +5,23 @@ // spell-checker:ignore (ToDO) gethostid -use clap::{crate_version, Command}; -use libc::c_long; +use clap::Command; +use libc::{c_long, gethostid}; use uucore::{error::UResult, format_usage, help_about, help_usage}; const USAGE: &str = help_usage!("hostid.md"); const ABOUT: &str = help_about!("hostid.md"); -// currently rust libc interface doesn't include gethostid -extern "C" { - pub fn gethostid() -> c_long; -} - #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - uu_app().get_matches_from(args); + uu_app().try_get_matches_from(args)?; hostid(); Ok(()) } pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) diff --git a/src/uu/hostname/Cargo.toml b/src/uu/hostname/Cargo.toml index 7dd6eabe5ad..40b62ef51a0 100644 --- a/src/uu/hostname/Cargo.toml +++ b/src/uu/hostname/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_hostname" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "hostname ~ (uutils) display or set the host name of the current host" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/hostname" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/hostname.rs" @@ -19,6 +22,9 @@ clap = { workspace = true } hostname = { workspace = true, features = ["set"] } uucore = { workspace = true, features = ["wide"] } +[target.'cfg(any(target_os = "freebsd", target_os = "openbsd"))'.dependencies] +dns-lookup = { workspace = true } + [target.'cfg(target_os = "windows")'.dependencies] windows-sys = { workspace = true, features = [ "Win32_Networking_WinSock", diff --git a/src/uu/hostname/src/hostname.rs b/src/uu/hostname/src/hostname.rs index 6a318cb8c26..29b2bb6ba1e 100644 --- a/src/uu/hostname/src/hostname.rs +++ b/src/uu/hostname/src/hostname.rs @@ -3,14 +3,18 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) MAKEWORD addrs hashset +// spell-checker:ignore hashset Addrs addrs +#[cfg(not(any(target_os = "freebsd", target_os = "openbsd")))] use std::net::ToSocketAddrs; use std::str; use std::{collections::hash_set::HashSet, ffi::OsString}; use clap::builder::ValueParser; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; + +#[cfg(any(target_os = "freebsd", target_os = "openbsd"))] +use dns_lookup::lookup_host; use uucore::{ error::{FromIo, UResult}, @@ -30,7 +34,7 @@ static OPT_HOST: &str = "host"; mod wsa { use std::io; - use windows_sys::Win32::Networking::WinSock::{WSACleanup, WSAStartup, WSADATA}; + use windows_sys::Win32::Networking::WinSock::{WSACleanup, WSADATA, WSAStartup}; pub(super) struct WsaHandle(()); @@ -71,7 +75,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -121,13 +125,25 @@ fn display_hostname(matches: &ArgMatches) -> UResult<()> { .into_owned(); if matches.get_flag(OPT_IP_ADDRESS) { - // XXX: to_socket_addrs needs hostname:port so append a dummy port and remove it later. - // This was originally supposed to use std::net::lookup_host, but that seems to be - // deprecated. Perhaps we should use the dns-lookup crate? - let hostname = hostname + ":1"; - let addresses = hostname - .to_socket_addrs() - .map_err_context(|| "failed to resolve socket addresses".to_owned())?; + let addresses; + + #[cfg(not(any(target_os = "freebsd", target_os = "openbsd")))] + { + let hostname = hostname + ":1"; + let addrs = hostname + .to_socket_addrs() + .map_err_context(|| "failed to resolve socket addresses".to_owned())?; + addresses = addrs; + } + + // DNS reverse lookup via "hostname:1" does not work on FreeBSD and OpenBSD + // use dns-lookup crate instead + #[cfg(any(target_os = "freebsd", target_os = "openbsd"))] + { + let addrs: Vec = lookup_host(hostname.as_str()).unwrap(); + addresses = addrs; + } + let mut hashset = HashSet::new(); let mut output = String::new(); for addr in addresses { diff --git a/src/uu/id/Cargo.toml b/src/uu/id/Cargo.toml index 6d332acfc3c..449b9cf925e 100644 --- a/src/uu/id/Cargo.toml +++ b/src/uu/id/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_id" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "id ~ (uutils) display user and group information for USER" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/id" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/id.rs" diff --git a/src/uu/id/src/id.rs b/src/uu/id/src/id.rs index b8fb51d79db..314d12d6804 100644 --- a/src/uu/id/src/id.rs +++ b/src/uu/id/src/id.rs @@ -33,12 +33,12 @@ #![allow(non_camel_case_types)] #![allow(dead_code)] -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use std::ffi::CStr; use uucore::display::Quotable; use uucore::entries::{self, Group, Locate, Passwd}; use uucore::error::UResult; -use uucore::error::{set_exit_code, USimpleError}; +use uucore::error::{USimpleError, set_exit_code}; pub use uucore::libc; use uucore::libc::{getlogin, uid_t}; use uucore::line_ending::LineEnding; @@ -47,7 +47,15 @@ use uucore::{format_usage, help_about, help_section, help_usage, show_error}; macro_rules! cstr2cow { ($v:expr) => { - unsafe { CStr::from_ptr($v).to_string_lossy() } + unsafe { + let ptr = $v; + // Must be not null to call cstr2cow + if ptr.is_null() { + None + } else { + Some({ CStr::from_ptr(ptr) }.to_string_lossy()) + } + } }; } @@ -130,7 +138,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { selinux_supported: { #[cfg(feature = "selinux")] { - selinux::kernel_support() != selinux::KernelSupport::Unsupported + uucore::selinux::is_selinux_enabled() } #[cfg(not(feature = "selinux"))] { @@ -149,7 +157,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { if (state.nflag || state.rflag) && default_format && !state.cflag { return Err(USimpleError::new( 1, - "cannot print only names or real IDs in default format", + "printing only names or real IDs requires -u, -g, or -G", )); } if state.zflag && default_format && !state.cflag { @@ -166,33 +174,27 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { )); } - let delimiter = { - if state.zflag { - "\0".to_string() - } else { - " ".to_string() - } - }; + let delimiter = if state.zflag { "\0" } else { " " }; let line_ending = LineEnding::from_zero_flag(state.zflag); if state.cflag { - if state.selinux_supported { + return if state.selinux_supported { // print SElinux context and exit #[cfg(all(any(target_os = "linux", target_os = "android"), feature = "selinux"))] if let Ok(context) = selinux::SecurityContext::current(false) { let bytes = context.as_bytes(); - print!("{}{}", String::from_utf8_lossy(bytes), line_ending); + print!("{}{line_ending}", String::from_utf8_lossy(bytes)); } else { // print error because `cflag` was explicitly requested return Err(USimpleError::new(1, "can't get process context")); } - return Ok(()); + Ok(()) } else { - return Err(USimpleError::new( + Err(USimpleError::new( 1, "--context (-Z) works only on an SELinux-enabled kernel", - )); - } + )) + }; } for i in 0..=users.len() { @@ -230,10 +232,14 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { return Ok(()); } - let (uid, gid) = possible_pw.as_ref().map(|p| (p.uid, p.gid)).unwrap_or(( - if state.rflag { getuid() } else { geteuid() }, - if state.rflag { getgid() } else { getegid() }, - )); + let (uid, gid) = possible_pw.as_ref().map(|p| (p.uid, p.gid)).unwrap_or({ + let use_effective = !state.rflag && (state.uflag || state.gflag || state.gsflag); + if use_effective { + (geteuid(), getegid()) + } else { + (getuid(), getgid()) + } + }); state.ids = Some(Ids { uid, gid, @@ -246,7 +252,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { "{}", if state.nflag { entries::gid2grp(gid).unwrap_or_else(|_| { - show_error!("cannot find name for group ID {}", gid); + show_error!("cannot find name for group ID {gid}"); set_exit_code(1); gid.to_string() }) @@ -261,7 +267,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { "{}", if state.nflag { entries::uid2usr(uid).unwrap_or_else(|_| { - show_error!("cannot find name for user ID {}", uid); + show_error!("cannot find name for user ID {uid}"); set_exit_code(1); uid.to_string() }) @@ -286,7 +292,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { .map(|&id| { if state.nflag { entries::gid2grp(id).unwrap_or_else(|_| { - show_error!("cannot find name for group ID {}", id); + show_error!("cannot find name for group ID {id}"); set_exit_code(1); id.to_string() }) @@ -295,7 +301,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } }) .collect::>() - .join(&delimiter), + .join(delimiter), // NOTE: this is necessary to pass GNU's "tests/id/zero.sh": if state.zflag && state.user_specified && users.len() > 1 { "\0" @@ -320,10 +326,11 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) + .args_override_self(true) .arg( Arg::new(options::OPT_AUDIT) .short('A') @@ -396,6 +403,7 @@ pub fn uu_app() -> Command { Arg::new(options::OPT_PASSWORD) .short('P') .help("Display the id as a password file entry.") + .conflicts_with(options::OPT_HUMAN_READABLE) .action(ArgAction::SetTrue), ) .arg( @@ -446,11 +454,11 @@ fn pretty(possible_pw: Option) { .join(" ") ); } else { - let login = cstr2cow!(getlogin() as *const _); + let login = cstr2cow!(getlogin().cast_const()); let rid = getuid(); if let Ok(p) = Passwd::locate(rid) { - if login == p.name { - println!("login\t{login}"); + if let Some(user_name) = login { + println!("login\t{user_name}"); } println!("uid\t{}", p.name); } else { @@ -555,40 +563,37 @@ fn id_print(state: &State, groups: &[u32]) { let egid = state.ids.as_ref().unwrap().egid; print!( - "uid={}({})", - uid, + "uid={uid}({})", entries::uid2usr(uid).unwrap_or_else(|_| { - show_error!("cannot find name for user ID {}", uid); + show_error!("cannot find name for user ID {uid}"); set_exit_code(1); uid.to_string() }) ); print!( - " gid={}({})", - gid, + " gid={gid}({})", entries::gid2grp(gid).unwrap_or_else(|_| { - show_error!("cannot find name for group ID {}", gid); + show_error!("cannot find name for group ID {gid}"); set_exit_code(1); gid.to_string() }) ); if !state.user_specified && (euid != uid) { print!( - " euid={}({})", - euid, + " euid={euid}({})", entries::uid2usr(euid).unwrap_or_else(|_| { - show_error!("cannot find name for user ID {}", euid); + show_error!("cannot find name for user ID {euid}"); set_exit_code(1); euid.to_string() }) ); } if !state.user_specified && (egid != gid) { + // BUG? printing egid={euid} ? print!( - " egid={}({})", - euid, + " egid={egid}({})", entries::gid2grp(egid).unwrap_or_else(|_| { - show_error!("cannot find name for group ID {}", egid); + show_error!("cannot find name for group ID {egid}"); set_exit_code(1); egid.to_string() }) @@ -599,10 +604,9 @@ fn id_print(state: &State, groups: &[u32]) { groups .iter() .map(|&gr| format!( - "{}({})", - gr, + "{gr}({})", entries::gid2grp(gr).unwrap_or_else(|_| { - show_error!("cannot find name for group ID {}", gr); + show_error!("cannot find name for group ID {gr}"); set_exit_code(1); gr.to_string() }) @@ -649,6 +653,7 @@ mod audit { pub type au_tid_addr_t = au_tid_addr; #[repr(C)] + #[expect(clippy::struct_field_names)] pub struct c_auditinfo_addr { pub ai_auid: au_id_t, // Audit user ID pub ai_mask: au_mask_t, // Audit masks. @@ -658,7 +663,7 @@ mod audit { } pub type c_auditinfo_addr_t = c_auditinfo_addr; - extern "C" { + unsafe extern "C" { pub fn getaudit(auditinfo_addr: *mut c_auditinfo_addr_t) -> c_int; } } diff --git a/src/uu/install/Cargo.toml b/src/uu/install/Cargo.toml index 647e0958ee0..5e7c5c5df87 100644 --- a/src/uu/install/Cargo.toml +++ b/src/uu/install/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_install" -version = "0.0.24" -authors = ["Ben Eills ", "uutils developers"] -license = "MIT" description = "install ~ (uutils) copy files from SOURCE to DESTINATION (with specified attributes)" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/install" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/install.rs" @@ -19,8 +22,10 @@ clap = { workspace = true } filetime = { workspace = true } file_diff = { workspace = true } libc = { workspace = true } +thiserror = { workspace = true } uucore = { workspace = true, features = [ "backup-control", + "buf-copy", "fs", "mode", "perms", diff --git a/src/uu/install/src/install.rs b/src/uu/install/src/install.rs index 9955be7b292..4cad5d1fb57 100644 --- a/src/uu/install/src/install.rs +++ b/src/uu/install/src/install.rs @@ -7,27 +7,30 @@ mod mode; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; use file_diff::diff; -use filetime::{set_file_times, FileTime}; -use std::error::Error; -use std::fmt::{Debug, Display}; -use std::fs; +use filetime::{FileTime, set_file_times}; +use std::fmt::Debug; use std::fs::File; -use std::os::unix::fs::MetadataExt; -#[cfg(unix)] -use std::os::unix::prelude::OsStrExt; -use std::path::{Path, PathBuf, MAIN_SEPARATOR}; +use std::fs::{self, metadata}; +use std::path::{MAIN_SEPARATOR, Path, PathBuf}; use std::process; +use thiserror::Error; use uucore::backup_control::{self, BackupMode}; +use uucore::buf_copy::copy_stream; use uucore::display::Quotable; use uucore::entries::{grp2gid, usr2uid}; -use uucore::error::{FromIo, UError, UIoError, UResult, UUsageError}; +use uucore::error::{FromIo, UError, UResult, UUsageError}; use uucore::fs::dir_strip_dot_for_creation; use uucore::mode::get_umask; -use uucore::perms::{wrap_chown, Verbosity, VerbosityLevel}; +use uucore::perms::{Verbosity, VerbosityLevel, wrap_chown}; use uucore::process::{getegid, geteuid}; -use uucore::{format_usage, help_about, help_usage, show, show_error, show_if_err, uio_error}; +use uucore::{format_usage, help_about, help_usage, show, show_error, show_if_err}; + +#[cfg(unix)] +use std::os::unix::fs::{FileTypeExt, MetadataExt}; +#[cfg(unix)] +use std::os::unix::prelude::OsStrExt; const DEFAULT_MODE: u32 = 0o755; const DEFAULT_STRIP_PROGRAM: &str = "strip"; @@ -47,25 +50,64 @@ pub struct Behavior { strip_program: String, create_leading: bool, target_dir: Option, + no_target_dir: bool, } -#[derive(Debug)] +#[derive(Error, Debug)] enum InstallError { + #[error("Unimplemented feature: {0}")] Unimplemented(String), - DirNeedsArg(), - CreateDirFailed(PathBuf, std::io::Error), + + #[error("{} with -d requires at least one argument.", uucore::util_name())] + DirNeedsArg, + + #[error("failed to create {0}")] + CreateDirFailed(PathBuf, #[source] std::io::Error), + + #[error("failed to chmod {}", .0.quote())] ChmodFailed(PathBuf), + + #[error("failed to chown {}: {}", .0.quote(), .1)] ChownFailed(PathBuf, String), + + #[error("invalid target {}: No such file or directory", .0.quote())] InvalidTarget(PathBuf), + + #[error("target {} is not a directory", .0.quote())] TargetDirIsntDir(PathBuf), - BackupFailed(PathBuf, PathBuf, std::io::Error), - InstallFailed(PathBuf, PathBuf, std::io::Error), + + #[error("cannot backup {0} to {1}")] + BackupFailed(PathBuf, PathBuf, #[source] std::io::Error), + + #[error("cannot install {0} to {1}")] + InstallFailed(PathBuf, PathBuf, #[source] std::io::Error), + + #[error("strip program failed: {0}")] StripProgramFailed(String), - MetadataFailed(std::io::Error), + + #[error("metadata error")] + MetadataFailed(#[source] std::io::Error), + + #[error("invalid user: {}", .0.quote())] InvalidUser(String), + + #[error("invalid group: {}", .0.quote())] InvalidGroup(String), + + #[error("omitting directory {}", .0.quote())] OmittingDirectory(PathBuf), + + #[error("failed to access {}: Not a directory", .0.quote())] NotADirectory(PathBuf), + + #[error("cannot overwrite directory {} with non-directory {}", .0.quote(), .1.quote())] + OverrideDirectoryFailed(PathBuf, PathBuf), + + #[error("'{0}' and '{1}' are the same file")] + SameFile(PathBuf, PathBuf), + + #[error("extra operand {}\n{}", .0.quote(), .1.quote())] + ExtraOperand(String, String), } impl UError for InstallError { @@ -81,52 +123,6 @@ impl UError for InstallError { } } -impl Error for InstallError {} - -impl Display for InstallError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Unimplemented(opt) => write!(f, "Unimplemented feature: {opt}"), - Self::DirNeedsArg() => { - write!( - f, - "{} with -d requires at least one argument.", - uucore::util_name() - ) - } - Self::CreateDirFailed(dir, e) => { - Display::fmt(&uio_error!(e, "failed to create {}", dir.quote()), f) - } - Self::ChmodFailed(file) => write!(f, "failed to chmod {}", file.quote()), - Self::ChownFailed(file, msg) => write!(f, "failed to chown {}: {}", file.quote(), msg), - Self::InvalidTarget(target) => write!( - f, - "invalid target {}: No such file or directory", - target.quote() - ), - Self::TargetDirIsntDir(target) => { - write!(f, "target {} is not a directory", target.quote()) - } - Self::BackupFailed(from, to, e) => Display::fmt( - &uio_error!(e, "cannot backup {} to {}", from.quote(), to.quote()), - f, - ), - Self::InstallFailed(from, to, e) => Display::fmt( - &uio_error!(e, "cannot install {} to {}", from.quote(), to.quote()), - f, - ), - Self::StripProgramFailed(msg) => write!(f, "strip program failed: {msg}"), - Self::MetadataFailed(e) => Display::fmt(&uio_error!(e, ""), f), - Self::InvalidUser(user) => write!(f, "invalid user: {}", user.quote()), - Self::InvalidGroup(group) => write!(f, "invalid group: {}", group.quote()), - Self::OmittingDirectory(dir) => write!(f, "omitting directory {}", dir.quote()), - Self::NotADirectory(dir) => { - write!(f, "failed to access {}: Not a directory", dir.quote()) - } - } - } -} - #[derive(Clone, Eq, PartialEq)] pub enum MainFunction { /// Create directories @@ -138,10 +134,7 @@ pub enum MainFunction { impl Behavior { /// Determine the mode for chmod after copy. pub fn mode(&self) -> u32 { - match self.specified_mode { - Some(x) => x, - None => DEFAULT_MODE, - } + self.specified_mode.unwrap_or(DEFAULT_MODE) } } @@ -191,7 +184,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -224,7 +217,6 @@ pub fn uu_app() -> Command { .action(ArgAction::SetTrue), ) .arg( - // TODO implement flag Arg::new(OPT_CREATE_LEADING) .short('D') .help( @@ -281,7 +273,6 @@ pub fn uu_app() -> Command { ) .arg(backup_control::arguments::suffix()) .arg( - // TODO implement flag Arg::new(OPT_TARGET_DIRECTORY) .short('t') .long(OPT_TARGET_DIRECTORY) @@ -290,11 +281,10 @@ pub fn uu_app() -> Command { .value_hint(clap::ValueHint::DirPath), ) .arg( - // TODO implement flag Arg::new(OPT_NO_TARGET_DIRECTORY) .short('T') .long(OPT_NO_TARGET_DIRECTORY) - .help("(unimplemented) treat DEST as a normal file") + .help("treat DEST as a normal file") .action(ArgAction::SetTrue), ) .arg( @@ -339,9 +329,7 @@ pub fn uu_app() -> Command { /// /// fn check_unimplemented(matches: &ArgMatches) -> UResult<()> { - if matches.get_flag(OPT_NO_TARGET_DIRECTORY) { - Err(InstallError::Unimplemented(String::from("--no-target-directory, -T")).into()) - } else if matches.get_flag(OPT_PRESERVE_CONTEXT) { + if matches.get_flag(OPT_PRESERVE_CONTEXT) { Err(InstallError::Unimplemented(String::from("--preserve-context, -P")).into()) } else if matches.get_flag(OPT_CONTEXT) { Err(InstallError::Unimplemented(String::from("--context, -Z")).into()) @@ -370,7 +358,7 @@ fn behavior(matches: &ArgMatches) -> UResult { let specified_mode: Option = if matches.contains_id(OPT_MODE) { let x = matches.get_one::(OPT_MODE).ok_or(1)?; Some(mode::parse(x, considering_dir, get_umask()).map_err(|err| { - show_error!("Invalid mode string: {}", err); + show_error!("Invalid mode string: {err}"); 1 })?) } else { @@ -379,6 +367,11 @@ fn behavior(matches: &ArgMatches) -> UResult { let backup_mode = backup_control::determine_backup_mode(matches)?; let target_dir = matches.get_one::(OPT_TARGET_DIRECTORY).cloned(); + let no_target_dir = matches.get_flag(OPT_NO_TARGET_DIRECTORY); + if target_dir.is_some() && no_target_dir { + show_error!("Options --target-directory and --no-target-directory are mutually exclusive"); + return Err(1.into()); + } let preserve_timestamps = matches.get_flag(OPT_PRESERVE_TIMESTAMPS); let compare = matches.get_flag(OPT_COMPARE); @@ -441,6 +434,7 @@ fn behavior(matches: &ArgMatches) -> UResult { ), create_leading: matches.get_flag(OPT_CREATE_LEADING), target_dir, + no_target_dir, }) } @@ -453,7 +447,7 @@ fn behavior(matches: &ArgMatches) -> UResult { /// fn directory(paths: &[String], b: &Behavior) -> UResult<()> { if paths.is_empty() { - Err(InstallError::DirNeedsArg().into()) + Err(InstallError::DirNeedsArg.into()) } else { for path in paths.iter().map(Path::new) { // if the path already exist, don't try to create it again @@ -533,6 +527,9 @@ fn standard(mut paths: Vec, b: &Behavior) -> UResult<()> { if paths.is_empty() { return Err(UUsageError::new(1, "missing file operand")); } + if b.no_target_dir && paths.len() > 2 { + return Err(InstallError::ExtraOperand(paths[2].clone(), format_usage(USAGE)).into()); + } // get the target from either "-t foo" param or from the last given paths argument let target: PathBuf = if let Some(path) = &b.target_dir { @@ -602,7 +599,7 @@ fn standard(mut paths: Vec, b: &Behavior) -> UResult<()> { } } - if sources.len() > 1 || is_potential_directory_path(&target) { + if sources.len() > 1 { copy_files_into_dir(sources, &target, b) } else { let source = sources.first().unwrap(); @@ -611,6 +608,16 @@ fn standard(mut paths: Vec, b: &Behavior) -> UResult<()> { return Err(InstallError::OmittingDirectory(source.clone()).into()); } + if b.no_target_dir && target.exists() { + return Err( + InstallError::OverrideDirectoryFailed(target.clone(), source.clone()).into(), + ); + } + + if is_potential_directory_path(&target) { + return copy_files_into_dir(sources, &target, b); + } + if target.is_file() || is_new_file_path(&target) { copy(source, &target, b) } else { @@ -649,7 +656,7 @@ fn copy_files_into_dir(files: &[PathBuf], target_dir: &Path, b: &Behavior) -> UR } let mut targetpath = target_dir.to_path_buf(); - let filename = sourcepath.components().last().unwrap(); + let filename = sourcepath.components().next_back().unwrap(); targetpath.push(filename); show_if_err!(copy(sourcepath, &targetpath, b)); @@ -692,7 +699,7 @@ fn chown_optional_user_group(path: &Path, b: &Behavior) -> UResult<()> { return Ok(()); }; - let meta = match fs::metadata(path) { + let meta = match metadata(path) { Ok(meta) => meta, Err(e) => return Err(InstallError::MetadataFailed(e).into()), }; @@ -723,12 +730,9 @@ fn perform_backup(to: &Path, b: &Behavior) -> UResult> { } let backup_path = backup_control::get_backup_path(b.backup_mode, to, &b.suffix); if let Some(ref backup_path) = backup_path { - // TODO!! - if let Err(err) = fs::rename(to, backup_path) { - return Err( - InstallError::BackupFailed(to.to_path_buf(), backup_path.clone(), err).into(), - ); - } + fs::rename(to, backup_path).map_err(|err| { + InstallError::BackupFailed(to.to_path_buf(), backup_path.clone(), err) + })?; } Ok(backup_path) } else { @@ -736,7 +740,24 @@ fn perform_backup(to: &Path, b: &Behavior) -> UResult> { } } -/// Copy a file from one path to another. +/// Copy a non-special file using std::fs::copy. +/// +/// # Parameters +/// * `from` - The source file path. +/// * `to` - The destination file path. +/// +/// # Returns +/// +/// Returns an empty Result or an error in case of failure. +fn copy_normal_file(from: &Path, to: &Path) -> UResult<()> { + if let Err(err) = fs::copy(from, to) { + return Err(InstallError::InstallFailed(from.to_path_buf(), to.to_path_buf(), err).into()); + } + Ok(()) +} + +/// Copy a file from one path to another. Handles the certain cases of special +/// files (e.g character specials). /// /// # Parameters /// @@ -748,18 +769,50 @@ fn perform_backup(to: &Path, b: &Behavior) -> UResult> { /// Returns an empty Result or an error in case of failure. /// fn copy_file(from: &Path, to: &Path) -> UResult<()> { - if from.as_os_str() == "/dev/null" { - /* workaround a limitation of fs::copy - * https://github.com/rust-lang/rust/issues/79390 - */ - if let Err(err) = File::create(to) { + if let Ok(to_abs) = to.canonicalize() { + if from.canonicalize()? == to_abs { + return Err(InstallError::SameFile(from.to_path_buf(), to.to_path_buf()).into()); + } + } + + if to.is_dir() && !from.is_dir() { + return Err(InstallError::OverrideDirectoryFailed( + to.to_path_buf().clone(), + from.to_path_buf().clone(), + ) + .into()); + } + // fs::copy fails if destination is a invalid symlink. + // so lets just remove all existing files at destination before copy. + if let Err(e) = fs::remove_file(to) { + if e.kind() != std::io::ErrorKind::NotFound { + show_error!( + "Failed to remove existing file {}. Error: {e:?}", + to.display(), + ); + } + } + + let ft = match metadata(from) { + Ok(ft) => ft.file_type(), + Err(err) => { return Err( InstallError::InstallFailed(from.to_path_buf(), to.to_path_buf(), err).into(), ); } - } else if let Err(err) = fs::copy(from, to) { - return Err(InstallError::InstallFailed(from.to_path_buf(), to.to_path_buf(), err).into()); + }; + + // Stream-based copying to get around the limitations of std::fs::copy + #[cfg(unix)] + if ft.is_char_device() || ft.is_block_device() || ft.is_fifo() { + let mut handle = File::open(from)?; + let mut dest = File::create(to)?; + copy_stream(&mut handle, &mut dest)?; + return Ok(()); } + + copy_normal_file(from, to)?; + Ok(()) } @@ -840,7 +893,7 @@ fn set_ownership_and_permissions(to: &Path, b: &Behavior) -> UResult<()> { /// Returns an empty Result or an error in case of failure. /// fn preserve_timestamps(from: &Path, to: &Path) -> UResult<()> { - let meta = match fs::metadata(from) { + let meta = match metadata(from) { Ok(meta) => meta, Err(e) => return Err(InstallError::MetadataFailed(e).into()), }; @@ -848,13 +901,11 @@ fn preserve_timestamps(from: &Path, to: &Path) -> UResult<()> { let modified_time = FileTime::from_last_modification_time(&meta); let accessed_time = FileTime::from_last_access_time(&meta); - match set_file_times(to, accessed_time, modified_time) { - Ok(_) => Ok(()), - Err(e) => { - show_error!("{}", e); - Ok(()) - } + if let Err(e) = set_file_times(to, accessed_time, modified_time) { + show_error!("{e}"); + // ignore error } + Ok(()) } /// Copy one file to a new location, changing metadata. @@ -921,16 +972,14 @@ fn copy(from: &Path, to: &Path, b: &Behavior) -> UResult<()> { fn need_copy(from: &Path, to: &Path, b: &Behavior) -> UResult { // Attempt to retrieve metadata for the source file. // If this fails, assume the file needs to be copied. - let from_meta = match fs::metadata(from) { - Ok(meta) => meta, - Err(_) => return Ok(true), + let Ok(from_meta) = metadata(from) else { + return Ok(true); }; // Attempt to retrieve metadata for the destination file. // If this fails, assume the file needs to be copied. - let to_meta = match fs::metadata(to) { - Ok(meta) => meta, - Err(_) => return Ok(true), + let Ok(to_meta) = metadata(to) else { + return Ok(true); }; // Define special file mode bits (setuid, setgid, sticky). diff --git a/src/uu/install/src/mode.rs b/src/uu/install/src/mode.rs index ebdec14afe6..5fcb9f332ee 100644 --- a/src/uu/install/src/mode.rs +++ b/src/uu/install/src/mode.rs @@ -25,7 +25,7 @@ pub fn chmod(path: &Path, mode: u32) -> Result<(), ()> { use std::os::unix::fs::PermissionsExt; use uucore::{display::Quotable, show_error}; fs::set_permissions(path, fs::Permissions::from_mode(mode)).map_err(|err| { - show_error!("{}: chmod failed with error {}", path.maybe_quote(), err); + show_error!("{}: chmod failed with error {err}", path.maybe_quote()); }) } diff --git a/src/uu/join/BENCHMARKING.md b/src/uu/join/BENCHMARKING.md index 80814ed1cb7..988259aa734 100644 --- a/src/uu/join/BENCHMARKING.md +++ b/src/uu/join/BENCHMARKING.md @@ -55,7 +55,7 @@ The following options can have a non-trivial impact on performance: - `-a`/`-v` if one of the two files has significantly more lines than the other - `-j`/`-1`/`-2` cause work to be done to grab the appropriate field -- `-i` adds a call to `to_ascii_lowercase()` that adds some time for allocating and dropping memory for the lowercase key +- `-i` uses our custom code for case-insensitive text comparisons - `--nocheck-order` causes some calls of `Input::compare` to be skipped The content of the files being joined has a very significant impact on the performance. diff --git a/src/uu/join/Cargo.toml b/src/uu/join/Cargo.toml index 759b04af712..1c7349c2c14 100644 --- a/src/uu/join/Cargo.toml +++ b/src/uu/join/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_join" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "join ~ (uutils) merge lines from inputs with matching join fields" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/join" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/join.rs" @@ -18,6 +21,7 @@ path = "src/join.rs" clap = { workspace = true } uucore = { workspace = true } memchr = { workspace = true } +thiserror = { workspace = true } [[bin]] name = "join" diff --git a/src/uu/join/src/join.rs b/src/uu/join/src/join.rs index 423af983ec9..0c6816cb649 100644 --- a/src/uu/join/src/join.rs +++ b/src/uu/join/src/join.rs @@ -3,69 +3,155 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) autoformat FILENUM whitespaces pairable unpairable nocheck +// spell-checker:ignore (ToDO) autoformat FILENUM whitespaces pairable unpairable nocheck memmem use clap::builder::ValueParser; -use clap::{crate_version, Arg, ArgAction, Command}; -use memchr::{memchr3_iter, memchr_iter}; +use clap::{Arg, ArgAction, Command}; +use memchr::{Memchr3, memchr_iter, memmem::Finder}; use std::cmp::Ordering; -use std::convert::From; -use std::error::Error; use std::ffi::OsString; -use std::fmt::Display; use std::fs::File; -use std::io::{stdin, stdout, BufRead, BufReader, BufWriter, Split, Stdin, Write}; +use std::io::{BufRead, BufReader, BufWriter, Split, Stdin, Write, stdin, stdout}; use std::num::IntErrorKind; #[cfg(unix)] use std::os::unix::ffi::OsStrExt; +use thiserror::Error; use uucore::display::Quotable; -use uucore::error::{set_exit_code, FromIo, UError, UResult, USimpleError}; +use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code}; use uucore::line_ending::LineEnding; -use uucore::{crash_if_err, format_usage, help_about, help_usage}; +use uucore::{format_usage, help_about, help_usage}; const ABOUT: &str = help_about!("join.md"); const USAGE: &str = help_usage!("join.md"); -#[derive(Debug)] +#[derive(Debug, Error)] enum JoinError { - IOError(std::io::Error), + #[error("io error: {0}")] + IOError(#[from] std::io::Error), + + #[error("{0}")] UnorderedInput(String), } +// If you still need the UError implementation for compatibility: impl UError for JoinError { fn code(&self) -> i32 { 1 } } -impl Error for JoinError {} +#[derive(Copy, Clone, PartialEq)] +enum FileNum { + File1, + File2, +} + +#[derive(Clone)] +enum SepSetting { + /// Any single-byte separator. + Byte(u8), + /// A single character more than one byte long. + Char(Vec), + /// No separators, join on the entire line. + Line, + /// Whitespace separators. + Whitespaces, +} + +trait Separator: Clone { + /// Using this separator, return the start and end index of all fields in the haystack. + fn field_ranges(&self, haystack: &[u8], len_guess: usize) -> Vec<(usize, usize)>; + /// The separator as it appears when in the output. + fn output_separator(&self) -> &[u8]; +} -impl Display for JoinError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::IOError(e) => write!(f, "io error: {e}"), - Self::UnorderedInput(e) => f.write_str(e), +/// Simple separators one byte in length. +#[derive(Copy, Clone)] +struct OneByteSep { + byte: [u8; 1], +} + +impl Separator for OneByteSep { + fn field_ranges(&self, haystack: &[u8], len_guess: usize) -> Vec<(usize, usize)> { + let mut field_ranges = Vec::with_capacity(len_guess); + let mut last_end = 0; + + for i in memchr_iter(self.byte[0], haystack) { + field_ranges.push((last_end, i)); + last_end = i + 1; } + field_ranges.push((last_end, haystack.len())); + field_ranges } + + fn output_separator(&self) -> &[u8] { + &self.byte + } +} + +/// Multi-byte (but still single character) separators. +#[derive(Clone)] +struct MultiByteSep<'a> { + finder: Finder<'a>, } -impl From for JoinError { - fn from(error: std::io::Error) -> Self { - Self::IOError(error) +impl Separator for MultiByteSep<'_> { + fn field_ranges(&self, haystack: &[u8], len_guess: usize) -> Vec<(usize, usize)> { + let mut field_ranges = Vec::with_capacity(len_guess); + let mut last_end = 0; + + for i in self.finder.find_iter(haystack) { + field_ranges.push((last_end, i)); + last_end = i + self.finder.needle().len(); + } + field_ranges.push((last_end, haystack.len())); + field_ranges + } + + fn output_separator(&self) -> &[u8] { + self.finder.needle() } } -#[derive(Copy, Clone, PartialEq)] -enum FileNum { - File1, - File2, +/// Whole-line separator. +#[derive(Copy, Clone)] +struct LineSep {} + +impl Separator for LineSep { + fn field_ranges(&self, haystack: &[u8], _len_guess: usize) -> Vec<(usize, usize)> { + vec![(0, haystack.len())] + } + + fn output_separator(&self) -> &[u8] { + &[] + } } -#[derive(Copy, Clone, PartialEq)] -enum Sep { - Char(u8), - Line, - Whitespaces, +/// Default whitespace separator. +#[derive(Copy, Clone)] +struct WhitespaceSep {} + +impl Separator for WhitespaceSep { + fn field_ranges(&self, haystack: &[u8], len_guess: usize) -> Vec<(usize, usize)> { + let mut field_ranges = Vec::with_capacity(len_guess); + let mut last_end = 0; + + // GNU join used Bourne shell field splitters by default + // FIXME: but now uses locale-dependent whitespace + for i in Memchr3::new(b' ', b'\t', b'\n', haystack) { + // leading whitespace should be dropped, contiguous whitespace merged + if i > last_end { + field_ranges.push((last_end, i)); + } + last_end = i + 1; + } + field_ranges.push((last_end, haystack.len())); + field_ranges + } + + fn output_separator(&self) -> &[u8] { + b" " + } } #[derive(Copy, Clone, PartialEq)] @@ -83,7 +169,7 @@ struct Settings { print_joined: bool, ignore_case: bool, line_ending: LineEnding, - separator: Sep, + separator: SepSetting, autoformat: bool, format: Vec, empty: Vec, @@ -101,7 +187,7 @@ impl Default for Settings { print_joined: true, ignore_case: false, line_ending: LineEnding::Newline, - separator: Sep::Whitespaces, + separator: SepSetting::Whitespaces, autoformat: false, format: vec![], empty: vec![], @@ -112,20 +198,15 @@ impl Default for Settings { } /// Output representation. -struct Repr<'a> { +struct Repr<'a, Sep: Separator> { line_ending: LineEnding, - separator: u8, - format: &'a [Spec], + separator: Sep, + format: Vec, empty: &'a [u8], } -impl<'a> Repr<'a> { - fn new( - line_ending: LineEnding, - separator: u8, - format: &'a [Spec], - empty: &'a [u8], - ) -> Repr<'a> { +impl<'a, Sep: Separator> Repr<'a, Sep> { + fn new(line_ending: LineEnding, separator: Sep, format: Vec, empty: &'a [u8]) -> Self { Repr { line_ending, separator, @@ -161,7 +242,7 @@ impl<'a> Repr<'a> { ) -> Result<(), std::io::Error> { for i in 0..line.field_ranges.len() { if i != index { - writer.write_all(&[self.separator])?; + writer.write_all(self.separator.output_separator())?; writer.write_all(line.get_field(i).unwrap())?; } } @@ -175,7 +256,7 @@ impl<'a> Repr<'a> { { for i in 0..self.format.len() { if i > 0 { - writer.write_all(&[self.separator])?; + writer.write_all(self.separator.output_separator())?; } let field = match f(&self.format[i]) { @@ -193,14 +274,48 @@ impl<'a> Repr<'a> { } } +/// Byte slice wrapper whose Ord implementation is case-insensitive on ASCII. +#[derive(Eq)] +struct CaseInsensitiveSlice<'a> { + v: &'a [u8], +} + +impl Ord for CaseInsensitiveSlice<'_> { + fn cmp(&self, other: &Self) -> Ordering { + if let Some((s, o)) = + std::iter::zip(self.v.iter(), other.v.iter()).find(|(s, o)| !s.eq_ignore_ascii_case(o)) + { + // first characters that differ, return the case-insensitive comparison + let s = s.to_ascii_lowercase(); + let o = o.to_ascii_lowercase(); + s.cmp(&o) + } else { + // one of the strings is a substring or equal of the other + self.v.len().cmp(&other.v.len()) + } + } +} + +impl PartialOrd for CaseInsensitiveSlice<'_> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for CaseInsensitiveSlice<'_> { + fn eq(&self, other: &Self) -> bool { + self.v.eq_ignore_ascii_case(other.v) + } +} + /// Input processing parameters. -struct Input { +struct Input { separator: Sep, ignore_case: bool, check_order: CheckOrder, } -impl Input { +impl Input { fn new(separator: Sep, ignore_case: bool, check_order: CheckOrder) -> Self { Self { separator, @@ -212,9 +327,9 @@ impl Input { fn compare(&self, field1: Option<&[u8]>, field2: Option<&[u8]>) -> Ordering { if let (Some(field1), Some(field2)) = (field1, field2) { if self.ignore_case { - field1 - .to_ascii_lowercase() - .cmp(&field2.to_ascii_lowercase()) + let field1 = CaseInsensitiveSlice { v: field1 }; + let field2 = CaseInsensitiveSlice { v: field2 }; + field1.cmp(&field2) } else { field1.cmp(field2) } @@ -277,24 +392,8 @@ struct Line { } impl Line { - fn new(string: Vec, separator: Sep, len_guess: usize) -> Self { - let mut field_ranges = Vec::with_capacity(len_guess); - let mut last_end = 0; - if separator == Sep::Whitespaces { - // GNU join uses Bourne shell field splitters by default - for i in memchr3_iter(b' ', b'\t', b'\n', &string) { - if i > last_end { - field_ranges.push((last_end, i)); - } - last_end = i + 1; - } - } else if let Sep::Char(sep) = separator { - for i in memchr_iter(sep, &string) { - field_ranges.push((last_end, i)); - last_end = i + 1; - } - } - field_ranges.push((last_end, string.len())); + fn new(string: Vec, separator: &Sep, len_guess: usize) -> Self { + let field_ranges = separator.field_ranges(&string, len_guess); Self { field_ranges, @@ -334,7 +433,7 @@ impl<'a> State<'a> { key: usize, line_ending: LineEnding, print_unpaired: bool, - ) -> UResult> { + ) -> UResult { let file_buf = if name == "-" { Box::new(stdin.lock()) as Box } else { @@ -357,7 +456,12 @@ impl<'a> State<'a> { } /// Skip the current unpaired line. - fn skip_line(&mut self, writer: &mut impl Write, input: &Input, repr: &Repr) -> UResult<()> { + fn skip_line( + &mut self, + writer: &mut impl Write, + input: &Input, + repr: &Repr<'a, Sep>, + ) -> UResult<()> { if self.print_unpaired { self.print_first_line(writer, repr)?; } @@ -368,7 +472,7 @@ impl<'a> State<'a> { /// Keep reading line sequence until the key does not change, return /// the first line whose key differs. - fn extend(&mut self, input: &Input) -> UResult> { + fn extend(&mut self, input: &Input) -> UResult> { while let Some(line) = self.next_line(input)? { let diff = input.compare(self.get_current_key(), line.get_field(self.key)); @@ -383,11 +487,11 @@ impl<'a> State<'a> { } /// Print lines in the buffers as headers. - fn print_headers( + fn print_headers( &self, writer: &mut impl Write, other: &State, - repr: &Repr, + repr: &Repr<'a, Sep>, ) -> Result<(), std::io::Error> { if self.has_line() { if other.has_line() { @@ -403,11 +507,11 @@ impl<'a> State<'a> { } /// Combine two line sequences. - fn combine( + fn combine( &self, writer: &mut impl Write, other: &State, - repr: &Repr, + repr: &Repr<'a, Sep>, ) -> Result<(), std::io::Error> { let key = self.get_current_key(); @@ -450,13 +554,16 @@ impl<'a> State<'a> { } } - fn reset_read_line(&mut self, input: &Input) -> Result<(), std::io::Error> { - let line = self.read_line(input.separator)?; + fn reset_read_line( + &mut self, + input: &Input, + ) -> Result<(), std::io::Error> { + let line = self.read_line(&input.separator)?; self.reset(line); Ok(()) } - fn reset_next_line(&mut self, input: &Input) -> Result<(), JoinError> { + fn reset_next_line(&mut self, input: &Input) -> Result<(), JoinError> { let line = self.next_line(input)?; self.reset(line); Ok(()) @@ -466,18 +573,27 @@ impl<'a> State<'a> { !self.seq.is_empty() } - fn initialize(&mut self, read_sep: Sep, autoformat: bool) -> usize { - if let Some(line) = crash_if_err!(1, self.read_line(read_sep)) { + fn initialize( + &mut self, + read_sep: &Sep, + autoformat: bool, + ) -> std::io::Result { + if let Some(line) = self.read_line(read_sep)? { self.seq.push(line); if autoformat { - return self.seq[0].field_ranges.len(); + return Ok(self.seq[0].field_ranges.len()); } } - 0 + Ok(0) } - fn finalize(&mut self, writer: &mut impl Write, input: &Input, repr: &Repr) -> UResult<()> { + fn finalize( + &mut self, + writer: &mut impl Write, + input: &Input, + repr: &Repr<'a, Sep>, + ) -> UResult<()> { if self.has_line() { if self.print_unpaired { self.print_first_line(writer, repr)?; @@ -497,7 +613,7 @@ impl<'a> State<'a> { } /// Get the next line without the order check. - fn read_line(&mut self, sep: Sep) -> Result, std::io::Error> { + fn read_line(&mut self, sep: &Sep) -> Result, std::io::Error> { match self.lines.next() { Some(value) => { self.line_num += 1; @@ -512,8 +628,8 @@ impl<'a> State<'a> { } /// Get the next line with the order check. - fn next_line(&mut self, input: &Input) -> Result, JoinError> { - if let Some(line) = self.read_line(input.separator)? { + fn next_line(&mut self, input: &Input) -> Result, JoinError> { + if let Some(line) = self.read_line(&input.separator)? { if input.check_order == CheckOrder::Disabled { return Ok(Some(line)); } @@ -534,7 +650,7 @@ impl<'a> State<'a> { if input.check_order == CheckOrder::Enabled { return Err(JoinError::UnorderedInput(err_msg)); } - eprintln!("{}: {}", uucore::execution_phrase(), err_msg); + eprintln!("{}: {err_msg}", uucore::execution_phrase()); self.has_failed = true; } @@ -549,11 +665,11 @@ impl<'a> State<'a> { self.seq[0].get_field(self.key) } - fn print_line( + fn print_line( &self, writer: &mut impl Write, line: &Line, - repr: &Repr, + repr: &Repr<'a, Sep>, ) -> Result<(), std::io::Error> { if repr.uses_format() { repr.print_format(writer, |spec| match *spec { @@ -574,32 +690,51 @@ impl<'a> State<'a> { repr.print_line_ending(writer) } - fn print_first_line(&self, writer: &mut impl Write, repr: &Repr) -> Result<(), std::io::Error> { + fn print_first_line( + &self, + writer: &mut impl Write, + repr: &Repr<'a, Sep>, + ) -> Result<(), std::io::Error> { self.print_line(writer, &self.seq[0], repr) } } -fn parse_separator(value_os: &OsString) -> UResult { +fn parse_separator(value_os: &OsString) -> UResult { + // Five possible separator values: + // No argument supplied, separate on whitespace; handled implicitly as the default elsewhere + // An empty string arg, whole line separation + // On unix-likes only, a single arbitrary byte + // The two-character "\0" string, interpreted as a single 0 byte + // A single scalar valid in the locale encoding (currently only UTF-8) + + if value_os.is_empty() { + return Ok(SepSetting::Line); + } + #[cfg(unix)] - let value = value_os.as_bytes(); - #[cfg(not(unix))] - let value = match value_os.to_str() { - Some(value) => value.as_bytes(), - None => { - return Err(USimpleError::new( - 1, - "unprintable field separators are only supported on unix-like platforms", - )); + { + let value = value_os.as_bytes(); + if value.len() == 1 { + return Ok(SepSetting::Byte(value[0])); } - }; - match value.len() { - 0 => Ok(Sep::Line), - 1 => Ok(Sep::Char(value[0])), - 2 if value[0] == b'\\' && value[1] == b'0' => Ok(Sep::Char(0)), - _ => Err(USimpleError::new( + } + + let Some(value) = value_os.to_str() else { + #[cfg(unix)] + return Err(USimpleError::new(1, "non-UTF-8 multi-byte tab")); + #[cfg(not(unix))] + return Err(USimpleError::new( 1, - format!("multi-character tab {}", value_os.to_string_lossy()), - )), + "unprintable field separators are only supported on unix-like platforms", + )); + }; + + let mut chars = value.chars(); + let c = chars.next().expect("valid string with at least one byte"); + match chars.next() { + None => Ok(SepSetting::Char(value.into())), + Some('0') if c == '\\' => Ok(SepSetting::Byte(0)), + _ => Err(USimpleError::new(1, format!("multi-character tab {value}"))), } } @@ -660,7 +795,7 @@ fn parse_settings(matches: &clap::ArgMatches) -> UResult { settings.autoformat = true; } else { let mut specs = vec![]; - for part in format.split(|c| c == ' ' || c == ',' || c == '\t') { + for part in format.split([' ', ',', '\t']) { specs.push(Spec::parse(part)?); } settings.format = specs; @@ -701,12 +836,25 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { return Err(USimpleError::new(1, "both files cannot be standard input")); } - exec(file1, file2, settings) + let sep = settings.separator.clone(); + match sep { + SepSetting::Byte(byte) => exec(file1, file2, settings, OneByteSep { byte: [byte] }), + SepSetting::Char(c) => exec( + file1, + file2, + settings, + MultiByteSep { + finder: Finder::new(&c), + }, + ), + SepSetting::Whitespaces => exec(file1, file2, settings, WhitespaceSep {}), + SepSetting::Line => exec(file1, file2, settings, LineSep {}), + } } pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -822,7 +970,7 @@ FILENUM is 1 or 2, corresponding to FILE1 or FILE2", ) } -fn exec(file1: &str, file2: &str, settings: Settings) -> UResult<()> { +fn exec(file1: &str, file2: &str, settings: Settings, sep: Sep) -> UResult<()> { let stdin = stdin(); let mut state1 = State::new( @@ -843,40 +991,29 @@ fn exec(file1: &str, file2: &str, settings: Settings) -> UResult<()> { settings.print_unpaired2, )?; - let input = Input::new( - settings.separator, - settings.ignore_case, - settings.check_order, - ); + let input = Input::new(sep.clone(), settings.ignore_case, settings.check_order); let format = if settings.autoformat { let mut format = vec![Spec::Key]; - let mut initialize = |state: &mut State| { - let max_fields = state.initialize(settings.separator, settings.autoformat); + let mut initialize = |state: &mut State| -> UResult<()> { + let max_fields = state.initialize(&sep, settings.autoformat)?; for i in 0..max_fields { if i != state.key { format.push(Spec::Field(state.file_num, i)); } } + Ok(()) }; - initialize(&mut state1); - initialize(&mut state2); + initialize(&mut state1)?; + initialize(&mut state2)?; format } else { - state1.initialize(settings.separator, settings.autoformat); - state2.initialize(settings.separator, settings.autoformat); + state1.initialize(&sep, settings.autoformat)?; + state2.initialize(&sep, settings.autoformat)?; settings.format }; - let repr = Repr::new( - settings.line_ending, - match settings.separator { - Sep::Char(sep) => sep, - _ => b' ', - }, - &format, - &settings.empty, - ); + let repr = Repr::new(settings.line_ending, sep, format, &settings.empty); let stdout = stdout(); let mut writer = BufWriter::new(stdout.lock()); diff --git a/src/uu/kill/Cargo.toml b/src/uu/kill/Cargo.toml index e60faa1c455..65385e28915 100644 --- a/src/uu/kill/Cargo.toml +++ b/src/uu/kill/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_kill" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "kill ~ (uutils) send a signal to a process" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/kill" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/kill.rs" diff --git a/src/uu/kill/src/kill.rs b/src/uu/kill/src/kill.rs index 87de0ff54d3..8d8aa0b614d 100644 --- a/src/uu/kill/src/kill.rs +++ b/src/uu/kill/src/kill.rs @@ -5,18 +5,23 @@ // spell-checker:ignore (ToDO) signalname pids killpg -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; use std::io::Error; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError}; -use uucore::signals::{signal_by_name_or_value, ALL_SIGNALS}; +use uucore::signals::{ALL_SIGNALS, signal_by_name_or_value, signal_name_by_value}; use uucore::{format_usage, help_about, help_usage, show}; static ABOUT: &str = help_about!("kill.md"); const USAGE: &str = help_usage!("kill.md"); +// When the -l option is selected, the program displays the type of signal related to a certain +// value or string. In case of a value, the program should control the lower 8 bits, but there is +// a particular case in which if the value is in range [128, 159], it is translated to a signal +const OFFSET: usize = 128; + pub mod options { pub static PIDS_OR_SIGNALS: &str = "pids_or_signals"; pub static LIST: &str = "list"; @@ -60,24 +65,45 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } else { 15_usize //SIGTERM }; - let sig: Signal = (sig as i32) - .try_into() - .map_err(|e| std::io::Error::from_raw_os_error(e as i32))?; + + let sig_name = signal_name_by_value(sig); + // Signal does not support converting from EXIT + // Instead, nix::signal::kill expects Option::None to properly handle EXIT + let sig: Option = if sig_name.is_some_and(|name| name == "EXIT") { + None + } else { + let sig = (sig as i32) + .try_into() + .map_err(|e| Error::from_raw_os_error(e as i32))?; + Some(sig) + }; + let pids = parse_pids(&pids_or_signals)?; - kill(sig, &pids); - Ok(()) + if pids.is_empty() { + Err(USimpleError::new( + 1, + "no process ID specified\n\ + Try --help for more information.", + )) + } else { + kill(sig, &pids); + Ok(()) + } } Mode::Table => { table(); Ok(()) } - Mode::List => list(pids_or_signals.first()), + Mode::List => { + list(&pids_or_signals); + Ok(()) + } } } pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -101,9 +127,11 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::SIGNAL) .short('s') + .short_alias('n') // For bash compatibility, like in GNU coreutils .long(options::SIGNAL) .value_name("signal") - .help("Sends given signal instead of SIGTERM"), + .help("Sends given signal instead of SIGTERM") + .conflicts_with_all([options::LIST, options::TABLE]), ) .arg( Arg::new(options::PIDS_OR_SIGNALS) @@ -118,6 +146,10 @@ fn handle_obsolete(args: &mut Vec) -> Option { // Old signal can only be in the first argument position let slice = args[1].as_str(); if let Some(signal) = slice.strip_prefix('-') { + // With '-', a signal name must start with an uppercase char + if signal.chars().next().is_some_and(|c| c.is_lowercase()) { + return None; + } // Check if it is a valid signal let opt_signal = signal_by_name_or_value(signal); if opt_signal.is_some() { @@ -131,49 +163,54 @@ fn handle_obsolete(args: &mut Vec) -> Option { } fn table() { - let name_width = ALL_SIGNALS.iter().map(|n| n.len()).max().unwrap(); - for (idx, signal) in ALL_SIGNALS.iter().enumerate() { - print!("{0: >#2} {1: <#2$}", idx, signal, name_width + 2); - if (idx + 1) % 7 == 0 { - println!(); - } + println!("{idx: >#2} {signal}"); } - println!(); } fn print_signal(signal_name_or_value: &str) -> UResult<()> { + // Closure used to track the last 8 bits of the signal value + // when the -l option is passed only the lower 8 bits are important + // or the value is in range [128, 159] + // Example: kill -l 143 => TERM because 143 = 15 + 128 + // Example: kill -l 2304 => EXIT + let lower_8_bits = |x: usize| x & 0xff; + let option_num_parse = signal_name_or_value.parse::().ok(); + for (value, &signal) in ALL_SIGNALS.iter().enumerate() { - if signal == signal_name_or_value || (format!("SIG{signal}")) == signal_name_or_value { + if signal.eq_ignore_ascii_case(signal_name_or_value) + || format!("SIG{signal}").eq_ignore_ascii_case(signal_name_or_value) + { println!("{value}"); return Ok(()); - } else if signal_name_or_value == value.to_string() { + } else if signal_name_or_value == value.to_string() + || option_num_parse.is_some_and(|signal_value| lower_8_bits(signal_value) == value) + || option_num_parse.is_some_and(|signal_value| signal_value == value + OFFSET) + { println!("{signal}"); return Ok(()); } } Err(USimpleError::new( 1, - format!("unknown signal name {}", signal_name_or_value.quote()), + format!("{}: invalid signal", signal_name_or_value.quote()), )) } fn print_signals() { - for (idx, signal) in ALL_SIGNALS.iter().enumerate() { - if idx > 0 { - print!(" "); - } - print!("{signal}"); + for signal in ALL_SIGNALS { + println!("{signal}"); } - println!(); } -fn list(arg: Option<&String>) -> UResult<()> { - match arg { - Some(x) => print_signal(x), - None => { - print_signals(); - Ok(()) +fn list(signals: &Vec) { + if signals.is_empty() { + print_signals(); + } else { + for signal in signals { + if let Err(e) = print_signal(signal) { + uucore::show!(e); + } } } } @@ -184,7 +221,7 @@ fn parse_signal_value(signal_name: &str) -> UResult { Some(x) => Ok(x), None => Err(USimpleError::new( 1, - format!("unknown signal name {}", signal_name.quote()), + format!("{}: invalid signal", signal_name.quote()), )), } } @@ -193,17 +230,19 @@ fn parse_pids(pids: &[String]) -> UResult> { pids.iter() .map(|x| { x.parse::().map_err(|e| { - USimpleError::new(1, format!("failed to parse argument {}: {}", x.quote(), e)) + USimpleError::new(1, format!("failed to parse argument {}: {e}", x.quote())) }) }) .collect() } -fn kill(sig: Signal, pids: &[i32]) { +fn kill(sig: Option, pids: &[i32]) { for &pid in pids { if let Err(e) = signal::kill(Pid::from_raw(pid), sig) { - show!(Error::from_raw_os_error(e as i32) - .map_err_context(|| format!("sending signal to {pid} failed"))); + show!( + Error::from_raw_os_error(e as i32) + .map_err_context(|| format!("sending signal to {pid} failed")) + ); } } } diff --git a/src/uu/link/Cargo.toml b/src/uu/link/Cargo.toml index d3643c5dcf6..41366f5d5a8 100644 --- a/src/uu/link/Cargo.toml +++ b/src/uu/link/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_link" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "link ~ (uutils) create a hard (file system) link to FILE" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/link" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/link.rs" diff --git a/src/uu/link/src/link.rs b/src/uu/link/src/link.rs index 806e89828bb..31f1239d86c 100644 --- a/src/uu/link/src/link.rs +++ b/src/uu/link/src/link.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. use clap::builder::ValueParser; -use clap::{crate_version, Arg, Command}; +use clap::{Arg, Command}; use std::ffi::OsString; use std::fs::hard_link; use std::path::Path; @@ -35,7 +35,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) diff --git a/src/uu/ln/Cargo.toml b/src/uu/ln/Cargo.toml index ba97c84e365..47a492a43b1 100644 --- a/src/uu/ln/Cargo.toml +++ b/src/uu/ln/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_ln" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "ln ~ (uutils) create a (file system) link to TARGET" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/ln" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/ln.rs" @@ -17,6 +20,7 @@ path = "src/ln.rs" [dependencies] clap = { workspace = true } uucore = { workspace = true, features = ["backup-control", "fs"] } +thiserror = { workspace = true } [[bin]] name = "ln" diff --git a/src/uu/ln/ln.md b/src/uu/ln/ln.md index b2320d6c427..6bd6ee01619 100644 --- a/src/uu/ln/ln.md +++ b/src/uu/ln/ln.md @@ -7,7 +7,7 @@ ln [OPTION]... TARGET... DIRECTORY ln [OPTION]... -t DIRECTORY TARGET... ``` -Change file owner and group +Make links between files. ## After Help diff --git a/src/uu/ln/src/ln.rs b/src/uu/ln/src/ln.rs index a056ee256a1..3b8ff0d7069 100644 --- a/src/uu/ln/src/ln.rs +++ b/src/uu/ln/src/ln.rs @@ -5,7 +5,7 @@ // spell-checker:ignore (ToDO) srcpath targetpath EEXIST -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use uucore::display::Quotable; use uucore::error::{FromIo, UError, UResult}; use uucore::fs::{make_path_relative_to, paths_refer_to_same_file}; @@ -13,10 +13,9 @@ use uucore::{format_usage, help_about, help_section, help_usage, prompt_yes, sho use std::borrow::Cow; use std::collections::HashSet; -use std::error::Error; use std::ffi::OsString; -use std::fmt::Display; use std::fs; +use thiserror::Error; #[cfg(any(unix, target_os = "redox"))] use std::os::unix::fs::symlink; @@ -24,7 +23,7 @@ use std::os::unix::fs::symlink; use std::os::windows::fs::{symlink_dir, symlink_file}; use std::path::{Path, PathBuf}; use uucore::backup_control::{self, BackupMode}; -use uucore::fs::{canonicalize, MissingHandling, ResolveMode}; +use uucore::fs::{MissingHandling, ResolveMode, canonicalize}; pub struct Settings { overwrite: OverwriteMode, @@ -46,38 +45,25 @@ pub enum OverwriteMode { Force, } -#[derive(Debug)] +#[derive(Error, Debug)] enum LnError { + #[error("target {} is not a directory", _0.quote())] TargetIsDirectory(PathBuf), + + #[error("")] SomeLinksFailed, + + #[error("{} and {} are the same file", _0.quote(), _1.quote())] SameFile(PathBuf, PathBuf), + + #[error("missing destination file operand after {}", _0.quote())] MissingDestination(PathBuf), - ExtraOperand(OsString), -} -impl Display for LnError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::TargetIsDirectory(s) => write!(f, "target {} is not a directory", s.quote()), - Self::SameFile(s, d) => { - write!(f, "{} and {} are the same file", s.quote(), d.quote()) - } - Self::SomeLinksFailed => Ok(()), - Self::MissingDestination(s) => { - write!(f, "missing destination file operand after {}", s.quote()) - } - Self::ExtraOperand(s) => write!( - f, - "extra operand {}\nTry '{} --help' for more information.", - s.quote(), - uucore::execution_phrase() - ), - } - } + #[error("extra operand {}\nTry '{} --help' for more information.", + format!("{_0:?}").trim_matches('"'), _1)] + ExtraOperand(OsString, String), } -impl Error for LnError {} - impl UError for LnError { fn code(&self) -> i32 { 1 @@ -107,8 +93,7 @@ static ARG_FILES: &str = "files"; #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let after_help = format!( - "{}\n\n{}", - AFTER_HELP, + "{AFTER_HELP}\n\n{}", backup_control::BACKUP_CONTROL_LONG_HELP ); @@ -158,7 +143,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -284,7 +269,11 @@ fn exec(files: &[PathBuf], settings: &Settings) -> UResult<()> { return Err(LnError::MissingDestination(files[0].clone()).into()); } if files.len() > 2 { - return Err(LnError::ExtraOperand(files[2].clone().into()).into()); + return Err(LnError::ExtraOperand( + files[2].clone().into(), + uucore::execution_phrase().to_string(), + ) + .into()); } assert!(!files.is_empty()); @@ -301,45 +290,45 @@ fn link_files_in_dir(files: &[PathBuf], target_dir: &Path, settings: &Settings) let mut all_successful = true; for srcpath in files { - let targetpath = - if settings.no_dereference && matches!(settings.overwrite, OverwriteMode::Force) { - // In that case, we don't want to do link resolution - // We need to clean the target - if target_dir.is_symlink() { - if target_dir.is_file() { - if let Err(e) = fs::remove_file(target_dir) { - show_error!("Could not update {}: {}", target_dir.quote(), e); - }; - } - if target_dir.is_dir() { - // Not sure why but on Windows, the symlink can be - // considered as a dir - // See test_ln::test_symlink_no_deref_dir - if let Err(e) = fs::remove_dir(target_dir) { - show_error!("Could not update {}: {}", target_dir.quote(), e); - }; + let targetpath = if settings.no_dereference + && matches!(settings.overwrite, OverwriteMode::Force) + && target_dir.is_symlink() + { + // In that case, we don't want to do link resolution + // We need to clean the target + if target_dir.is_file() { + if let Err(e) = fs::remove_file(target_dir) { + show_error!("Could not update {}: {e}", target_dir.quote()); + }; + } + if target_dir.is_dir() { + // Not sure why but on Windows, the symlink can be + // considered as a dir + // See test_ln::test_symlink_no_deref_dir + if let Err(e) = fs::remove_dir(target_dir) { + show_error!("Could not update {}: {e}", target_dir.quote()); + }; + } + target_dir.to_path_buf() + } else { + match srcpath.as_os_str().to_str() { + Some(name) => { + match Path::new(name).file_name() { + Some(basename) => target_dir.join(basename), + // This can be None only for "." or "..". Trying + // to create a link with such name will fail with + // EEXIST, which agrees with the behavior of GNU + // coreutils. + None => target_dir.join(name), } } - target_dir.to_path_buf() - } else { - match srcpath.as_os_str().to_str() { - Some(name) => { - match Path::new(name).file_name() { - Some(basename) => target_dir.join(basename), - // This can be None only for "." or "..". Trying - // to create a link with such name will fail with - // EEXIST, which agrees with the behavior of GNU - // coreutils. - None => target_dir.join(name), - } - } - None => { - show_error!("cannot stat {}: No such file or directory", srcpath.quote()); - all_successful = false; - continue; - } + None => { + show_error!("cannot stat {}: No such file or directory", srcpath.quote()); + all_successful = false; + continue; } - }; + } + }; if linked_destinations.contains(&targetpath) { // If the target file was already created in this ln call, do not overwrite @@ -350,7 +339,7 @@ fn link_files_in_dir(files: &[PathBuf], target_dir: &Path, settings: &Settings) ); all_successful = false; } else if let Err(e) = link(srcpath, &targetpath, settings) { - show_error!("{}", e); + show_error!("{e}"); all_successful = false; } @@ -387,12 +376,12 @@ fn link(src: &Path, dst: &Path, settings: &Settings) -> UResult<()> { if dst.is_symlink() || dst.exists() { backup_path = match settings.backup { - BackupMode::NoBackup => None, - BackupMode::SimpleBackup => Some(simple_backup_path(dst, &settings.suffix)), - BackupMode::NumberedBackup => Some(numbered_backup_path(dst)), - BackupMode::ExistingBackup => Some(existing_backup_path(dst, &settings.suffix)), + BackupMode::None => None, + BackupMode::Simple => Some(simple_backup_path(dst, &settings.suffix)), + BackupMode::Numbered => Some(numbered_backup_path(dst)), + BackupMode::Existing => Some(existing_backup_path(dst, &settings.suffix)), }; - if settings.backup == BackupMode::ExistingBackup && !settings.symbolic { + if settings.backup == BackupMode::Existing && !settings.symbolic { // when ln --backup f f, it should detect that it is the same file if paths_refer_to_same_file(src, dst, true) { return Err(LnError::SameFile(src.to_owned(), dst.to_owned()).into()); diff --git a/src/uu/logname/Cargo.toml b/src/uu/logname/Cargo.toml index d9870b1d99c..e723595b3b4 100644 --- a/src/uu/logname/Cargo.toml +++ b/src/uu/logname/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_logname" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "logname ~ (uutils) display the login name of the current user" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/logname" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/logname.rs" diff --git a/src/uu/logname/src/logname.rs b/src/uu/logname/src/logname.rs index 02a78cf4c3c..5437bbae344 100644 --- a/src/uu/logname/src/logname.rs +++ b/src/uu/logname/src/logname.rs @@ -5,11 +5,11 @@ // spell-checker:ignore (ToDO) getlogin userlogin -use clap::{crate_version, Command}; +use clap::Command; use std::ffi::CStr; use uucore::{error::UResult, format_usage, help_about, help_usage, show_error}; -extern "C" { +unsafe extern "C" { // POSIX requires using getlogin (or equivalent code) pub fn getlogin() -> *const libc::c_char; } @@ -42,7 +42,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .override_usage(format_usage(USAGE)) .about(ABOUT) .infer_long_args(true) diff --git a/src/uu/ls/BENCHMARKING.md b/src/uu/ls/BENCHMARKING.md index 3103e43b0f1..3611d7a9608 100644 --- a/src/uu/ls/BENCHMARKING.md +++ b/src/uu/ls/BENCHMARKING.md @@ -1,6 +1,13 @@ # Benchmarking ls ls majorly involves fetching a lot of details (depending upon what details are requested, eg. time/date, inode details, etc) for each path using system calls. Ideally, any system call should be done only once for each of the paths - not adhering to this principle leads to a lot of system call overhead multiplying and bubbling up, especially for recursive ls, therefore it is important to always benchmark multiple scenarios. + +ls _also_ prints a lot of information, so optimizing formatting operations is also critical: + - Try to avoid using `format` unless required. + - Try to avoid repeated string copies unless necessary. + - If a temporary buffer is required, try to allocate a reasonable capacity to start with to avoid repeated reallocations. + - Some values might be expensive to compute (e.g. current line width), but are only required in some cases. Consider delaying computations, e.g. by wrapping the evaluation in a LazyCell. + This is an overview over what was benchmarked, and if you make changes to `ls`, you are encouraged to check how performance was affected for the workloads listed below. Feel free to add other workloads to the list that we should improve / make sure not to regress. diff --git a/src/uu/ls/Cargo.toml b/src/uu/ls/Cargo.toml index 49c64ba0937..ff00175e747 100644 --- a/src/uu/ls/Cargo.toml +++ b/src/uu/ls/Cargo.toml @@ -1,39 +1,45 @@ [package] name = "uu_ls" -version = "0.0.24" -authors = ["uutils developers"] -license = "MIT" description = "ls ~ (uutils) display directory contents" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/ls" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/ls.rs" [dependencies] -clap = { workspace = true, features = ["env"] } +ansi-width = { workspace = true } chrono = { workspace = true } -unicode-width = { workspace = true } -number_prefix = { workspace = true } -uutils_term_grid = { workspace = true } -terminal_size = { workspace = true } +clap = { workspace = true, features = ["env"] } glob = { workspace = true } +hostname = { workspace = true } lscolors = { workspace = true } +number_prefix = { workspace = true } +selinux = { workspace = true, optional = true } +terminal_size = { workspace = true } +thiserror = { workspace = true } uucore = { workspace = true, features = [ "colors", + "custom-tz-fmt", "entries", + "format", "fs", "fsxattr", + "parser", "quoting-style", "version-cmp", ] } -once_cell = { workspace = true } -selinux = { workspace = true, optional = true } -hostname = { workspace = true } +uutils_term_grid = { workspace = true } [[bin]] name = "ls" diff --git a/src/uu/ls/src/colors.rs b/src/uu/ls/src/colors.rs new file mode 100644 index 00000000000..ab19672ea98 --- /dev/null +++ b/src/uu/ls/src/colors.rs @@ -0,0 +1,202 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +use super::PathData; +use super::get_metadata_with_deref_opt; +use lscolors::{Indicator, LsColors, Style}; +use std::ffi::OsString; +use std::fs::{DirEntry, Metadata}; +use std::io::{BufWriter, Stdout}; + +/// We need this struct to be able to store the previous style. +/// This because we need to check the previous value in case we don't need +/// the reset +pub(crate) struct StyleManager<'a> { + /// last style that is applied, if `None` that means reset is applied. + pub(crate) current_style: Option