From 66565dbe58a2dfb35aff055a0210c93b279b7818 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Wed, 28 May 2025 10:47:10 -0500 Subject: [PATCH 01/23] Output poison leak as message instead of warning (#847) --- .../CheckForPoison.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs b/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs index 470fafac704..61e21fec912 100644 --- a/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs +++ b/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs @@ -171,7 +171,7 @@ public override bool Execute() } else if (poisons.Count() > 0) { - Log.LogWarning($"{poisons.Count()} marked files leaked to output. See complete report '{PoisonReportOutputFilePath}' for details."); + Log.LogMessage($"{poisons.Count()} marked files leaked to output. See complete report '{PoisonReportOutputFilePath}' for details."); } return !Log.HasLoggedErrors; From 0798aada21870867ecce6510856707b6e6f7163b Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 28 May 2025 09:28:19 -0700 Subject: [PATCH 02/23] Move BuildComparer from VersionTools to build manifest package (#848) --- Directory.Packages.props | 2 +- eng/Version.Details.xml | 2 +- eng/Versions.props | 2 +- eng/tools/BuildComparer/AssetComparer.cs | 3 +-- eng/tools/BuildComparer/BuildComparer.cs | 4 ++-- eng/tools/BuildComparer/BuildComparer.csproj | 2 +- eng/tools/BuildComparer/Program.cs | 3 +-- eng/tools/BuildComparer/Utils.cs | 3 +-- 8 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 28b7331b6d7..d54bcf6c82d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9d0d6868bda..881016eb1c7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,7 +6,7 @@ https://github.com/dotnet/dotnet b6887ba7da1721be2dcd30d68126d4a725be47f0 - + https://github.com/dotnet/dotnet b6887ba7da1721be2dcd30d68126d4a725be47f0 diff --git a/eng/Versions.props b/eng/Versions.props index cb970997247..1bac51cf735 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -26,7 +26,7 @@ 10.0.100-preview.6.25272.112 10.0.100-preview.6.25272.112 - 10.0.0-beta.25272.112 + 10.0.0-beta.25272.112 2.0.0-beta5.25208.1 diff --git a/eng/tools/BuildComparer/AssetComparer.cs b/eng/tools/BuildComparer/AssetComparer.cs index 692678d6f99..482da248639 100644 --- a/eng/tools/BuildComparer/AssetComparer.cs +++ b/eng/tools/BuildComparer/AssetComparer.cs @@ -1,8 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.VersionTools.Automation; -using Microsoft.DotNet.VersionTools.BuildManifest; +using Microsoft.DotNet.Build.Manifest; using NuGet.Packaging; using System.Collections.Concurrent; using System.Collections.Immutable; diff --git a/eng/tools/BuildComparer/BuildComparer.cs b/eng/tools/BuildComparer/BuildComparer.cs index a07a24622ca..080fbf95a32 100644 --- a/eng/tools/BuildComparer/BuildComparer.cs +++ b/eng/tools/BuildComparer/BuildComparer.cs @@ -1,8 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.VersionTools.Automation; -using Microsoft.DotNet.VersionTools.BuildManifest; +using Microsoft.Arcade.Common; +using Microsoft.DotNet.Build.Manifest; using NuGet.Packaging; using System.Collections.Concurrent; using System.Collections.Immutable; diff --git a/eng/tools/BuildComparer/BuildComparer.csproj b/eng/tools/BuildComparer/BuildComparer.csproj index 1cfd60e535e..0dbe36eb7ee 100644 --- a/eng/tools/BuildComparer/BuildComparer.csproj +++ b/eng/tools/BuildComparer/BuildComparer.csproj @@ -7,7 +7,7 @@ - + diff --git a/eng/tools/BuildComparer/Program.cs b/eng/tools/BuildComparer/Program.cs index 9da39dda436..fed7c52a641 100644 --- a/eng/tools/BuildComparer/Program.cs +++ b/eng/tools/BuildComparer/Program.cs @@ -1,8 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.VersionTools.Automation; -using Microsoft.DotNet.VersionTools.BuildManifest; +using Microsoft.DotNet.Build.Manifest; using NuGet.Packaging; using System.Collections.Concurrent; using System.Collections.Immutable; diff --git a/eng/tools/BuildComparer/Utils.cs b/eng/tools/BuildComparer/Utils.cs index d3f7362634c..9002464f2e8 100644 --- a/eng/tools/BuildComparer/Utils.cs +++ b/eng/tools/BuildComparer/Utils.cs @@ -1,8 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.VersionTools.Automation; -using Microsoft.DotNet.VersionTools.BuildManifest; +using Microsoft.DotNet.Build.Manifest; using NuGet.Packaging; using System.Collections.Concurrent; using System.Collections.Immutable; From 998688a2ac4773d76486300876cbb573337d4c33 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 28 May 2025 11:37:24 -0700 Subject: [PATCH 03/23] Set IsShipping on discovered assets during publishing (#850) --- eng/Publishing.props | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/Publishing.props b/eng/Publishing.props index 58b060691cf..7419cf7f96c 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -74,6 +74,8 @@ Blob + true + false %(ProducedAsset.Identity) DotNetReleaseShipping=%(ProducedAsset.DotNetReleaseShipping) From f94a58a2971030efc68cf56abfd47dad1260f6d7 Mon Sep 17 00:00:00 2001 From: William Godbe Date: Wed, 28 May 2025 15:03:31 -0700 Subject: [PATCH 04/23] Baseline 2 more missing assets (#849) --- eng/vmr-msft-comparison-baseline.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eng/vmr-msft-comparison-baseline.json b/eng/vmr-msft-comparison-baseline.json index 118ffe65509..bb6c4d38b98 100644 --- a/eng/vmr-msft-comparison-baseline.json +++ b/eng/vmr-msft-comparison-baseline.json @@ -328,6 +328,11 @@ "idMatch": ".*Microsoft.AspNetCore.Watch.BrowserRefresh.Tests.*", "justification": "sdk non-shipping test packages are not published" }, + { + "issueType": "MissingNonShipping", + "idMatch": ".*Microsoft.DotNet.ApiDiff.Tests.*", + "justification": "sdk non-shipping test packages are not published" + }, { "issueType": "MissingNonShipping", "idMatch": ".*Microsoft.DotNet.Cli.Utils.Tests.*", @@ -518,5 +523,10 @@ "idMatch": ".*/dotnet-sdk-pgo-.*.tar.gz", "descriptionMatch": ".* is Excluded in the VMR but should be Signed", "justification": "The VMR build purposely excludes signing the PGO builds." + }, + { + "issueType": "MissingShipping", + "idMatch": ".*dotnet-suggest.*", + "justification": "command-line-api is not built from the VMR" } ] From 93b312fad74f04e4f82694a13dea189780dc700c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 29 May 2025 15:34:29 +0200 Subject: [PATCH 05/23] Remove broken Codespaces scenarios (#871) --- .devcontainer/README.md | 32 ++------------------ .devcontainer/init.sh | 2 -- .devcontainer/prebuilt-sdk/devcontainer.json | 22 -------------- .devcontainer/prebuilt-sdk/init.sh | 9 ------ .devcontainer/synchronize-vmr.sh | 4 --- 5 files changed, 2 insertions(+), 67 deletions(-) delete mode 100644 .devcontainer/prebuilt-sdk/devcontainer.json delete mode 100644 .devcontainer/prebuilt-sdk/init.sh delete mode 100755 .devcontainer/synchronize-vmr.sh diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 7f0a5cd2646..0b57c52abed 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -12,12 +12,10 @@ This Codespace allows you to debug or make changes to the .NET SDK product. The 45 up to 75 minutes (depending on the machine and OS) and, after completion, produces an archived .NET SDK located in `/workspaces/dotnet/artifacts/assets/Release`. -In case you selected the prebuilt-sdk Codespace configuration (which is only useful when building -from source), the built-from-source SDK will already be there. - ## Build the SDK To build the repository, run one of the following: + ```bash # Microsoft based build ./build.sh @@ -32,30 +30,4 @@ or > Please note that, at this time, the build modifies some of the checked-in sources so it might be preferential to rebuild the Codespace between attempts (or reset the working tree changes). -For more details, see the instructions at https://github.com/dotnet/dotnet. - -## Synchronize your changes in locally - -When debugging the build, you have two options how to test your changes in this environment. - -### Making changes to the VMR directly - -You can make the changes directly to the local checkout of the VMR at `/workspaces/dotnet`. You -can then try to build the VMR and see if the change works for you. - -### Pull changes into the Codespace from your fork - -You can also make a fix in the individual source repository (e.g. `dotnet/runtime`) and push the -fix into a branch; can be in your fork too. Once you have the commit pushed, you can pull this -version of the repository into the Codespace by running: - -``` -/workspaces/synchronize-vmr.sh \ - --repository : \ - --remote : -``` - -You can now proceed building the VMR in the Codespace using instructions above. You can repeat -this process and sync a new commit from your fork. Only note that, at this time, Source-Build -modifies some of the checked-in sources so you'll need to revert the working tree changes -between attempts. +For more details, see the instructions at https://github.com/dotnet/dotnet#building. diff --git a/.devcontainer/init.sh b/.devcontainer/init.sh index c7315cf3e74..e329e79c358 100755 --- a/.devcontainer/init.sh +++ b/.devcontainer/init.sh @@ -11,6 +11,4 @@ vmr_dir=$(realpath "$workspace_dir/dotnet") $vmr_dir/.devcontainer/init-toolset.sh $vmr_dir -cp "$vmr_dir/.devcontainer/synchronize-vmr.sh" "$workspace_dir" - mkdir -p "$tmp_dir" diff --git a/.devcontainer/prebuilt-sdk/devcontainer.json b/.devcontainer/prebuilt-sdk/devcontainer.json deleted file mode 100644 index c30ad3f24f0..00000000000 --- a/.devcontainer/prebuilt-sdk/devcontainer.json +++ /dev/null @@ -1,22 +0,0 @@ -// Container contains a pre-built SDK -{ - "name": "Pre-built .NET SDK", - "image": "mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9", - "hostRequirements": { - // A completely source built .NET is >64 GB with all the repos/artifacts - "storage": "128gb" - }, - "customizations": { - "vscode": { - "extensions": [ - "ms-dotnettools.csharp" - ] - }, - "codespaces": { - "openFiles": [ - ".devcontainer/README.md" - ] - } - }, - "onCreateCommand": ".devcontainer/prebuilt-sdk/init.sh" -} diff --git a/.devcontainer/prebuilt-sdk/init.sh b/.devcontainer/prebuilt-sdk/init.sh deleted file mode 100644 index 517ed62296b..00000000000 --- a/.devcontainer/prebuilt-sdk/init.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -script_root="$( cd -P "$( dirname "$source" )" && pwd )" - -"$script_root"/../../prep-source-build.sh - -cp "$script_root/../synchronize-vmr.sh" "/workspaces/" -"$script_root"/../../build.sh --online --clean-while-building || exit 0 diff --git a/.devcontainer/synchronize-vmr.sh b/.devcontainer/synchronize-vmr.sh deleted file mode 100755 index ce3a2a74291..00000000000 --- a/.devcontainer/synchronize-vmr.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -(cd /workspaces/dotnet/src/sdk \ - && ./eng/vmr-sync.sh --vmr /workspaces/dotnet --tmp /workspaces/tmp $*) From 56a60bd9357d6a543f758629a46f741b2c7214bf Mon Sep 17 00:00:00 2001 From: Ella Hathaway <67609881+ellahathaway@users.noreply.github.com> Date: Thu, 29 May 2025 09:03:09 -0700 Subject: [PATCH 06/23] Add validation for changes to protected files (#760) --- .../workflows/protected-files-validation.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/protected-files-validation.yml diff --git a/.github/workflows/protected-files-validation.yml b/.github/workflows/protected-files-validation.yml new file mode 100644 index 00000000000..6fac13eb5fa --- /dev/null +++ b/.github/workflows/protected-files-validation.yml @@ -0,0 +1,22 @@ +name: 'Check Protected Files for Changes' + +on: + pull_request: + # Update the error message when changing these paths + paths: + - eng/common/**/* + - eng/Version.Details.xml + - eng/Versions.props + +permissions: + pull-requests: read + +jobs: + check-protect-files: + runs-on: ubuntu-22.04 + if: ${{ github.event.pull_request.user.login != 'dotnet-sb-bot' && github.event.pull_request.user.login != 'dotnet-maestro' }} + steps: + - name: Protected File has Changes + run: | + echo "Cannot make changes to 'eng/Version.Details.xml', 'eng/Versions.props', and 'eng/common/' outside of a rebootstrap or .NET Source-Build release PR." + exit 1 From 97a81c029cdd3edff96496de79a3944eff4b0db0 Mon Sep 17 00:00:00 2001 From: Ella Hathaway <67609881+ellahathaway@users.noreply.github.com> Date: Thu, 29 May 2025 10:35:06 -0700 Subject: [PATCH 07/23] Exclude `libbootstrapper*.o` files from signing validation (#824) --- eng/SignCheckExclusionsFile.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/SignCheckExclusionsFile.txt b/eng/SignCheckExclusionsFile.txt index c6c7f14c7d5..5d4a5146c6c 100644 --- a/eng/SignCheckExclusionsFile.txt +++ b/eng/SignCheckExclusionsFile.txt @@ -24,6 +24,7 @@ ILCompiler.Build.Tasks.dll;; IGNORE-STRONG-NAME, https://github.com/dotnet/sourc ;; ## MACH-O ## *.dwarf;; Debugging symbols, https://github.com/dotnet/source-build/issues/4994#issuecomment-2768803544 +libbootstrapper*.o;; Not executables, https://github.com/dotnet/runtime/issues/114701#issuecomment-2807546040 ;; ## ESRP Signing Issues ## *.exe;*.whl;The .whl files are not supported by ESRP, https://github.com/dotnet/emsdk/commit/3ac196ac425223d11490e759c2641256ce746f21 From 92e8051969ec2ce456ec0e80b479ae58e42dc5e1 Mon Sep 17 00:00:00 2001 From: Michael Simons Date: Thu, 29 May 2025 15:58:16 -0500 Subject: [PATCH 08/23] Updates to readme's filing issues and useful links (#872) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Přemek Vysoký --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 82d5a7a26d1..bf36936c82a 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,11 @@ In the VMR, you can find: - *[in future]* E2E tests for the whole .NET product. Just like the development repositories, the VMR will have a release branch for every feature band (e.g. `release/10.0.1xx`). -Similarly, VMR's `main` branch will follow default branches of product repositories (see [Synchronization Based on Declared Dependencies](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md#synchronization-based-on-declared-dependencies)). +Similarly, VMR's `main` branch will follow default branches of product repositories (see [Synchronization Based on Declared Dependencies](docs/VMR-Design-And-Operation.md#synchronization-based-on-declared-dependencies)). -More in-depth documentation about the VMR can be found in [VMR Design And Operation](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md#layout). +More in-depth documentation about the VMR can be found in [VMR Design And Operation](docs/VMR-Design-And-Operation.md#layout). See also [dotnet/source-build](https://github.com/dotnet/source-build) for more information about our whole-product source-build. - ## Installing the SDK You can download the .NET SDK either as an installer (MSI, PKG) or as an archive (zip, tar.gz). The .NET SDK contains both the .NET runtimes and CLI tools. @@ -32,7 +31,7 @@ You can download the .NET SDK either as an installer (MSI, PKG) or as an archive - The main purpose of the [dotnet/dotnet](https://github.com/dotnet/dotnet) repository is to have all source code necessary to build the .NET product available in one repository and identified by a single commit. - The VMR also aims to become the place from which we release and service future versions of .NET to reduce the complexity of the product construction process. This should allow our partners and and 3rd parties to easily build, test and modify .NET using their custom infrastructure as well as make the process available to the community. - Lastly, we hope to solve other problems that the current multi-repo setup brings: - - Enable the standard [down-/up-stream open-source model](src/arcade/Documentation/UnifiedBuild/VMR-Upstream-Downstream.md). + - Enable the standard [down-/up-stream open-source model](docs/VMR-Upstream-Downstream.md). - Fulfill requirements of .NET distro builders such as RedHat or Canonical to natively include .NET in their distribution repositories. - Simplify scenarios such as client-run testing of bug fixes and improvements. The build should work in an offline environment too for certain platforms. - Enable developers to make and test changes spanning multiple repositories. @@ -199,20 +198,21 @@ Alternatively, you can also provide a manifest file where this information can b Sometimes you want to make a change in a repository and test that change in the VMR. You could of course make the change in the VMR directly, but in case it's already available in your repository, you can synchronize it locally into your clone of the VMR, commit, and then open a PR. -To do this, you can either start a [dotnet/dotnet](https://github.com/dotnet/dotnet) Codespace - you will see instructions right after it starts. Alternatively, you can clone the repository locally and use the [vmr-sync.sh](src/sdk/eng/vmr-sync.sh) or [vmr-sync.ps1](src/sdk/eng/vmr-sync.ps1) script to pull your changes in. Please refer to the documentation in the script for more details. +To do this, you need to use the [`darc vmr forwardflow` command](https://github.com/dotnet/arcade-services/blob/main/docs/Darc.md#forwardflow) which can move your changes from your repository's dev branch into a local VMR one. Please refer to command's documentation (`--help`) for more details. ## Filing Issues -This repo does not currently accept issues. Please file issues to the appropriate development repos. -For issues with the VMR itself, please use the [source-build repository](https://github.com/dotnet/source-build). +This repo should contain issues that are tied to the VMR infrastructure and documentation. + +For other issues, please open them in the appropriate product repos. We have links to many of them on [our new issue page](https://github.com/dotnet/dotnet/issues/new/choose). ## Useful Links - Design documentation for the VMR - a set of documents describing the high-level design and the why's and how's - - [Design and Operation](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md) - - [Upstream/Downstream Relationships](src/arcade/Documentation/UnifiedBuild/VMR-Upstream-Downstream.md) - - [Code and Build Workflow](src/arcade/Documentation/UnifiedBuild/VMR-Code-And-Build-Workflow.md) - - [Strategy for Managing External Source Dependencies](src/arcade/Documentation/UnifiedBuild/VMR-Strategy-For-External-Source.md) + - [Design and Operation](docs/VMR-Design-And-Operation.md) + - [Upstream/Downstream Relationships](docs/VMR-Upstream-Downstream.md) + - [Code and Build Workflow](docs/VMR-Code-And-Build-Workflow.md) + - [Strategy for Managing External Source Dependencies](docs/VMR-Strategy-For-External-Source.md) - [.NET Source-Build](https://github.com/dotnet/source-build) - [What is .NET](https://dotnet.microsoft.com) From c08d049becd41f07374d73be307a0e5f9a4ede7b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 00:48:17 +0000 Subject: [PATCH 09/23] [main] Source code updates from dotnet/source-build-externals (#864) Co-authored-by: dotnet-maestro[bot] Co-authored-by: Viktor Hofer --- src/source-build-externals/eng/Version.Details.xml | 6 +++--- src/source-build-externals/eng/common/build.sh | 2 +- src/source-build-externals/eng/common/tools.ps1 | 1 - src/source-build-externals/eng/common/tools.sh | 2 -- src/source-build-externals/global.json | 2 +- src/source-manifest.json | 4 ++-- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/source-build-externals/eng/Version.Details.xml b/src/source-build-externals/eng/Version.Details.xml index ff4f3ae84ba..379d1b33874 100644 --- a/src/source-build-externals/eng/Version.Details.xml +++ b/src/source-build-externals/eng/Version.Details.xml @@ -1,14 +1,14 @@ - + https://github.com/dotnet/msbuild e9b99f554a3c298e1106ea171f5a0462780af2c5 - + https://github.com/dotnet/dotnet - f5705c8f4c5079bba77bae8698ba1583bde0388c + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d diff --git a/src/source-build-externals/eng/common/build.sh b/src/source-build-externals/eng/common/build.sh index b105db583c4..9767bb411a4 100755 --- a/src/source-build-externals/eng/common/build.sh +++ b/src/source-build-externals/eng/common/build.sh @@ -257,7 +257,7 @@ function Build { /p:Sign=$sign \ /p:Publish=$publish \ /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \ - "${properties[@]}" + ${properties[@]+"${properties[@]}"} ExitWithExitCode 0 } diff --git a/src/source-build-externals/eng/common/tools.ps1 b/src/source-build-externals/eng/common/tools.ps1 index 046ec9d08a0..c9e39595b58 100644 --- a/src/source-build-externals/eng/common/tools.ps1 +++ b/src/source-build-externals/eng/common/tools.ps1 @@ -644,7 +644,6 @@ function GetNuGetPackageCachePath() { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' - $env:RESTORENOHTTPCACHE = $true } } diff --git a/src/source-build-externals/eng/common/tools.sh b/src/source-build-externals/eng/common/tools.sh index 8bc68e8460f..28944dfcb3f 100755 --- a/src/source-build-externals/eng/common/tools.sh +++ b/src/source-build-externals/eng/common/tools.sh @@ -345,14 +345,12 @@ function InitializeBuildTool { _InitializeBuildToolCommand="msbuild" } -# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages/" else export NUGET_PACKAGES="$repo_root/.packages/" - export RESTORENOHTTPCACHE=true fi fi diff --git a/src/source-build-externals/global.json b/src/source-build-externals/global.json index fbc9f4083a0..1a44a561f47 100644 --- a/src/source-build-externals/global.json +++ b/src/source-build-externals/global.json @@ -3,7 +3,7 @@ "dotnet": "10.0.100-preview.6.25272.112" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25276.103", + "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25277.102", "Microsoft.Build.NoTargets": "3.7.0" } } diff --git a/src/source-manifest.json b/src/source-manifest.json index 56ffe592bd6..5ee1e9ce83d 100644 --- a/src/source-manifest.json +++ b/src/source-manifest.json @@ -128,10 +128,10 @@ }, { "packageVersion": "10.0.622801", - "barId": 269640, + "barId": 269843, "path": "source-build-externals", "remoteUri": "https://github.com/dotnet/source-build-externals", - "commitSha": "9c353b86754dc74d2508aaecca0d68549a601eab" + "commitSha": "21542951d5dba2f477782fe2af2b66a660b019d7" }, { "packageVersion": "", From b9041c05d7d614ff1f1ff2a691adf99284772704 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 00:49:15 +0000 Subject: [PATCH 10/23] [main] Source code updates from dotnet/runtime (#867) Co-authored-by: dotnet-maestro[bot] Co-authored-by: Viktor Hofer --- src/runtime/Directory.Build.targets | 1 - src/runtime/eng/python.targets | 31 -- .../scenarios/BuildWasmAppsJobsList.txt | 1 + .../System.Private.CoreLib.csproj | 27 +- src/runtime/src/coreclr/jit/abi.cpp | 23 + src/runtime/src/coreclr/jit/abi.h | 1 + .../src/coreclr/jit/codegenloongarch64.cpp | 10 + .../src/coreclr/jit/codegenriscv64.cpp | 10 + src/runtime/src/coreclr/jit/codegenxarch.cpp | 2 - src/runtime/src/coreclr/jit/compiler.h | 7 +- src/runtime/src/coreclr/jit/compiler.hpp | 68 +++ src/runtime/src/coreclr/jit/fgopt.cpp | 73 +-- src/runtime/src/coreclr/jit/fgprofile.cpp | 11 + .../src/coreclr/jit/fgprofilesynthesis.cpp | 11 + src/runtime/src/coreclr/jit/forwardsub.cpp | 13 +- src/runtime/src/coreclr/jit/gentree.cpp | 31 +- src/runtime/src/coreclr/jit/gentree.h | 16 +- src/runtime/src/coreclr/jit/gschecks.cpp | 34 +- src/runtime/src/coreclr/jit/instrsxarch.h | 12 +- src/runtime/src/coreclr/jit/lower.cpp | 186 +++++-- src/runtime/src/coreclr/jit/lower.h | 12 +- src/runtime/src/coreclr/jit/morph.cpp | 111 ++-- src/runtime/src/coreclr/jit/optcse.cpp | 210 ++++---- src/runtime/src/coreclr/jit/optcse.h | 8 +- src/runtime/src/coreclr/jit/optimizer.cpp | 52 +- .../src/System.Private.CoreLib.csproj | 28 +- .../src/coreclr/pal/src/exception/signal.cpp | 9 +- .../pal/tests/palsuite/compilableTests.txt | 21 - .../pal/tests/palsuite/paltestlist.txt | 2 - .../palsuite/paltestlist_to_be_reviewed.txt | 22 - .../coreclr/pal/tests/palsuite/paltests.cpp | 2 +- .../coreclr/scripts/genRuntimeEventSources.py | 484 ------------------ .../Runtime/RiscVLoongArch64FpStruct.cs | 48 +- .../ObjectWriter/R2RPEBuilder.cs | 108 +--- src/runtime/src/coreclr/vm/methodtable.cpp | 89 +++- ...Microsoft.Extensions.FileSystemGlobbing.cs | 1 + .../src/Internal/IncludeOrExcludeValue.cs | 11 + .../src/Internal/MatcherContext.cs | 127 ++--- .../CompositePatternContext.cs | 27 + .../IncludesFirstCompositePatternContext.cs | 87 ++++ .../PreserveOrderCompositePatternContext.cs | 65 +++ .../src/Matcher.cs | 49 +- .../tests/OrderedPatternMatchingTests.cs | 210 ++++++++ .../tests/PatternMatchingTests.cs | 95 ++-- .../FileSystemGlobbingTestContext.cs | 4 +- .../src/System/Collections/BitArray.cs | 2 +- .../src/System/Formats/Tar/GnuTarEntry.cs | 19 +- .../PaxGlobalExtendedAttributesTarEntry.cs | 2 +- .../src/System/Formats/Tar/PaxTarEntry.cs | 98 +--- .../src/System/Formats/Tar/TarHeader.Read.cs | 33 +- .../src/System/Formats/Tar/TarHeader.Write.cs | 21 +- .../src/System/Formats/Tar/TarHeader.cs | 6 +- .../TarEntry/GnuTarEntry.Conversion.Tests.cs | 2 + .../TarEntry/PaxTarEntry.Conversion.Tests.cs | 28 +- .../TarEntry.Conversion.Tests.Base.cs | 152 +++--- .../UstarTarEntry.Conversion.Tests.cs | 2 + .../TarEntry/V7TarEntry.Conversion.Tests.cs | 2 + .../tests/TarReader/TarReader.Tests.cs | 2 +- .../tests/TarTestsBase.Pax.cs | 20 +- .../System.Formats.Tar/tests/TarTestsBase.cs | 65 ++- .../TarWriter.WriteEntry.Entry.Pax.Tests.cs | 20 +- ...rWriter.WriteEntryAsync.Entry.Pax.Tests.cs | 18 +- .../gen/NativeRuntimeEventSourceGenerator.cs | 424 +++++++++++++++ .../System.Private.CoreLib.Generators.csproj | 4 +- .../src/System/MemoryExtensions.cs | 14 +- .../DefaultInterpolatedStringHandler.cs | 14 +- .../src/System/Text/StringBuilder.cs | 22 +- .../src/System/Text/Unicode/Utf8.cs | 14 +- .../ref/System.Reflection.TypeExtensions.cs | 4 +- .../src/System/Reflection/TypeExtensions.cs | 16 +- .../src/Recognition/RecognizerBase.cs | 2 +- .../System/Text/Json/Document/JsonProperty.cs | 14 +- .../DefaultJsonTypeInfoResolver.Helpers.cs | 2 +- ...iCompatBaseline.NetCoreAppLatestStable.xml | 12 + .../System.Private.CoreLib.csproj | 30 +- .../runtime/diagnostics/diagnostics-js.ts | 3 + .../Blazor/BlazorRunOptions.cs | 2 + .../Blazor/EventPipeDiagnosticsTests.cs | 346 +++++++++++++ .../BrowserStructures/RunOptions.cs | 1 + .../Templates/WasmTemplateTestsBase.cs | 2 +- .../Wasm.Build.Tests/Wasm.Build.Tests.csproj | 1 + .../wasm/host/DevServer/DevServerStartup.cs | 53 ++ .../App/BlazorBasicTestApp.csproj | 2 +- .../App/BlazorBasicTestApp.runtimeconfig.json | 13 + .../src/native/external/zlib-ng-version.txt | 11 +- src/runtime/src/native/external/zlib-ng.cmake | 7 + .../native/external/zlib-ng/.gitattributes | 15 +- .../src/native/external/zlib-ng/.gitignore | 8 + .../native/external/zlib-ng/CMakeLists.txt | 53 +- .../src/native/external/zlib-ng/Makefile.in | 2 +- .../src/native/external/zlib-ng/README.md | 3 +- .../native/external/zlib-ng/adler32_fold.c | 16 - .../native/external/zlib-ng/adler32_fold.h | 11 - .../external/zlib-ng/arch/arm/arm_functions.h | 4 +- .../external/zlib-ng/arch/arm/chunkset_neon.c | 19 +- .../zlib-ng/arch/arm/compare256_neon.c | 2 +- .../external/zlib-ng/arch/arm/crc32_acle.c | 42 +- .../zlib-ng/arch/arm/insert_string_acle.c | 24 - .../external/zlib-ng/arch/arm/neon_intrins.h | 20 +- .../external/zlib-ng/arch/generic/Makefile.in | 4 +- .../zlib-ng/arch/generic/chunkset_c.c | 12 +- .../zlib-ng/arch/generic/compare256_c.c | 180 +------ .../zlib-ng/arch/generic/compare256_p.h | 123 +++++ .../zlib-ng/arch/generic/generic_functions.h | 57 +-- .../zlib-ng/arch/power/chunkset_power8.c | 13 +- .../zlib-ng/arch/power/compare256_power9.c | 2 +- .../zlib-ng/arch/power/crc32_power8.c | 4 +- .../zlib-ng/arch/power/power_functions.h | 2 +- .../{fallback_builtins.h => power_intrins.h} | 7 +- .../external/zlib-ng/arch/riscv/adler32_rvv.c | 10 +- .../zlib-ng/arch/riscv/compare256_rvv.c | 2 +- .../zlib-ng/arch/riscv/riscv_features.c | 21 +- .../zlib-ng/arch/riscv/riscv_functions.h | 2 +- .../external/zlib-ng/arch/s390/README.md | 54 +- .../zlib-ng/arch/s390/dfltcc_common.c | 40 -- .../zlib-ng/arch/s390/s390_features.c | 2 +- .../zlib-ng/arch/s390/s390_functions.h | 7 + .../external/zlib-ng/arch/x86/Makefile.in | 13 +- .../zlib-ng/arch/x86/adler32_avx512_p.h | 11 + .../external/zlib-ng/arch/x86/avx2_tables.h | 44 ++ .../external/zlib-ng/arch/x86/chunkset_avx2.c | 97 ++-- .../zlib-ng/arch/x86/chunkset_avx512.c | 182 +++++++ .../external/zlib-ng/arch/x86/chunkset_sse2.c | 15 +- .../zlib-ng/arch/x86/chunkset_ssse3.c | 15 +- .../zlib-ng/arch/x86/compare256_avx2.c | 2 +- .../zlib-ng/arch/x86/compare256_sse2.c | 2 +- .../zlib-ng/arch/x86/insert_string_sse42.c | 24 - .../external/zlib-ng/arch/x86/x86_features.c | 4 +- .../external/zlib-ng/arch/x86/x86_features.h | 1 + .../external/zlib-ng/arch/x86/x86_functions.h | 15 +- .../external/zlib-ng/arch/x86/x86_intrins.h | 5 + .../src/native/external/zlib-ng/chunkset.c | 42 -- .../native/external/zlib-ng/chunkset_tpl.h | 175 +++++-- .../external/zlib-ng/cmake/detect-arch.c | 2 +- .../zlib-ng/cmake/detect-intrinsics.cmake | 16 +- .../zlib-ng/cmake/detect-sanitizer.cmake | 11 +- .../src/native/external/zlib-ng/compare256.c | 180 ------- .../native/external/zlib-ng/compare256_rle.h | 57 +-- .../src/native/external/zlib-ng/configure | 27 +- .../src/native/external/zlib-ng/crc32_braid.c | 267 ---------- .../src/native/external/zlib-ng/crc32_fold.c | 33 -- .../src/native/external/zlib-ng/crc32_fold.h | 21 - .../src/native/external/zlib-ng/deflate.c | 4 +- .../src/native/external/zlib-ng/deflate.h | 20 +- .../native/external/zlib-ng/deflate_quick.c | 2 +- .../src/native/external/zlib-ng/deflate_rle.c | 19 +- .../native/external/zlib-ng/deflate_slow.c | 2 +- .../src/native/external/zlib-ng/functable.c | 26 +- .../src/native/external/zlib-ng/functable.h | 2 +- .../src/native/external/zlib-ng/infback.c | 13 +- .../src/native/external/zlib-ng/inffast_tpl.h | 52 +- .../src/native/external/zlib-ng/inflate.c | 47 +- .../src/native/external/zlib-ng/inflate.h | 43 +- .../src/native/external/zlib-ng/inflate_p.h | 34 +- .../external/zlib-ng/insert_string_tpl.h | 9 +- .../src/native/external/zlib-ng/match_tpl.h | 62 +-- .../src/native/external/zlib-ng/slide_hash.c | 52 -- .../src/native/external/zlib-ng/trees_emit.h | 8 +- .../external/zlib-ng/win32/Makefile.a64 | 4 +- .../external/zlib-ng/win32/Makefile.arm | 4 +- .../external/zlib-ng/win32/Makefile.msc | 8 +- .../src/native/external/zlib-ng/zbuild.h | 50 +- .../src/native/external/zlib-ng/zlib-ng.h.in | 8 +- .../src/native/external/zlib-ng/zlib.h.in | 8 +- .../src/native/external/zlib-ng/zmemory.h | 99 ++++ .../src/native/external/zlib-ng/zutil.c | 2 +- .../src/native/external/zlib-ng/zutil_p.h | 29 -- src/runtime/src/tests/Directory.Build.targets | 4 +- .../JitBlue/Runtime_115109/Runtime_115109.cs | 46 ++ .../Runtime_115109/Runtime_115109.csproj | 8 + .../JIT/opt/SVE/PredicateInstructions.cs | 27 +- src/source-manifest.json | 4 +- 172 files changed, 3503 insertions(+), 3170 deletions(-) delete mode 100644 src/runtime/eng/python.targets delete mode 100644 src/runtime/src/coreclr/scripts/genRuntimeEventSources.py create mode 100644 src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/IncludeOrExcludeValue.cs create mode 100644 src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/CompositePatternContext.cs create mode 100644 src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/IncludesFirstCompositePatternContext.cs create mode 100644 src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/PreserveOrderCompositePatternContext.cs create mode 100644 src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/OrderedPatternMatchingTests.cs create mode 100644 src/runtime/src/libraries/System.Private.CoreLib/gen/NativeRuntimeEventSourceGenerator.cs create mode 100644 src/runtime/src/mono/wasm/Wasm.Build.Tests/Blazor/EventPipeDiagnosticsTests.cs create mode 100644 src/runtime/src/mono/wasm/testassets/BlazorBasicTestApp/App/BlazorBasicTestApp.runtimeconfig.json delete mode 100644 src/runtime/src/native/external/zlib-ng/adler32_fold.c delete mode 100644 src/runtime/src/native/external/zlib-ng/adler32_fold.h delete mode 100644 src/runtime/src/native/external/zlib-ng/arch/arm/insert_string_acle.c create mode 100644 src/runtime/src/native/external/zlib-ng/arch/generic/compare256_p.h rename src/runtime/src/native/external/zlib-ng/arch/power/{fallback_builtins.h => power_intrins.h} (91%) delete mode 100644 src/runtime/src/native/external/zlib-ng/arch/s390/dfltcc_common.c create mode 100644 src/runtime/src/native/external/zlib-ng/arch/x86/avx2_tables.h create mode 100644 src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_avx512.c delete mode 100644 src/runtime/src/native/external/zlib-ng/arch/x86/insert_string_sse42.c delete mode 100644 src/runtime/src/native/external/zlib-ng/chunkset.c delete mode 100644 src/runtime/src/native/external/zlib-ng/compare256.c delete mode 100644 src/runtime/src/native/external/zlib-ng/crc32_braid.c delete mode 100644 src/runtime/src/native/external/zlib-ng/crc32_fold.c delete mode 100644 src/runtime/src/native/external/zlib-ng/crc32_fold.h delete mode 100644 src/runtime/src/native/external/zlib-ng/slide_hash.c create mode 100644 src/runtime/src/native/external/zlib-ng/zmemory.h create mode 100644 src/runtime/src/tests/JIT/Regression/JitBlue/Runtime_115109/Runtime_115109.cs create mode 100644 src/runtime/src/tests/JIT/Regression/JitBlue/Runtime_115109/Runtime_115109.csproj diff --git a/src/runtime/Directory.Build.targets b/src/runtime/Directory.Build.targets index ae85cfcd844..827532ba468 100644 --- a/src/runtime/Directory.Build.targets +++ b/src/runtime/Directory.Build.targets @@ -26,7 +26,6 @@ - diff --git a/src/runtime/eng/python.targets b/src/runtime/eng/python.targets deleted file mode 100644 index 3f933fdfea0..00000000000 --- a/src/runtime/eng/python.targets +++ /dev/null @@ -1,31 +0,0 @@ - - - - <_PythonLocationScript>-c "import sys; sys.stdout.write(sys.executable)" - - - - - - - - - - - - - - diff --git a/src/runtime/eng/testing/scenarios/BuildWasmAppsJobsList.txt b/src/runtime/eng/testing/scenarios/BuildWasmAppsJobsList.txt index 3446a86a2b9..82c8b0c8279 100644 --- a/src/runtime/eng/testing/scenarios/BuildWasmAppsJobsList.txt +++ b/src/runtime/eng/testing/scenarios/BuildWasmAppsJobsList.txt @@ -15,6 +15,7 @@ Wasm.Build.Tests.Blazor.NativeTests Wasm.Build.Tests.Blazor.NoopNativeRebuildTest Wasm.Build.Tests.Blazor.WorkloadRequiredTests Wasm.Build.Tests.Blazor.SignalRClientTests +Wasm.Build.Tests.Blazor.EventPipeDiagnosticsTests Wasm.Build.Tests.BuildPublishTests Wasm.Build.Tests.ConfigSrcTests Wasm.Build.Tests.DllImportTests diff --git a/src/runtime/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/runtime/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj index 854e13fedb8..07861b33acb 100644 --- a/src/runtime/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj +++ b/src/runtime/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj @@ -320,30 +320,7 @@ - - - - - src\System\Diagnostics\Eventing\NativeRuntimeEventSource.Generated.cs - - + + - - - - - - <_PythonWarningParameter>-Wall - <_PythonWarningParameter Condition="'$(MSBuildTreatWarningsAsErrors)' == 'true'">$(_PythonWarningParameter) -Werror - <_EventingSourceFileDirectory>%(EventingSourceFile.RootDir)%(EventingSourceFile.Directory) - <_EventingSourceFileDirectory Condition="HasTrailingSlash('$(_EventingSourceFileDirectory)')">$(_EventingSourceFileDirectory.TrimEnd('\')) - - - - - - - - - diff --git a/src/runtime/src/coreclr/jit/abi.cpp b/src/runtime/src/coreclr/jit/abi.cpp index 0525e18888f..198f523f6b6 100644 --- a/src/runtime/src/coreclr/jit/abi.cpp +++ b/src/runtime/src/coreclr/jit/abi.cpp @@ -154,6 +154,29 @@ var_types ABIPassingSegment::GetRegisterType() const } } +//----------------------------------------------------------------------------- +// GetRegisterType: +// Return the smallest type larger or equal to Size that most naturally +// represents the register this segment is passed in, taking into account the +// GC info of the specified layout. +// +// Return Value: +// A type that matches ABIPassingSegment::Size and the register. +// +var_types ABIPassingSegment::GetRegisterType(ClassLayout* layout) const +{ + if (genIsValidIntReg(GetRegister())) + { + assert(Offset < layout->GetSize()); + if (((Offset % TARGET_POINTER_SIZE) == 0) && (Size == TARGET_POINTER_SIZE)) + { + return layout->GetGCPtrType(Offset / TARGET_POINTER_SIZE); + } + } + + return GetRegisterType(); +} + //----------------------------------------------------------------------------- // InRegister: // Create an ABIPassingSegment representing that a segment is passed in a diff --git a/src/runtime/src/coreclr/jit/abi.h b/src/runtime/src/coreclr/jit/abi.h index dcb7b73aacf..dcb23cfd374 100644 --- a/src/runtime/src/coreclr/jit/abi.h +++ b/src/runtime/src/coreclr/jit/abi.h @@ -41,6 +41,7 @@ class ABIPassingSegment unsigned GetStackSize() const; var_types GetRegisterType() const; + var_types GetRegisterType(ClassLayout* layout) const; static ABIPassingSegment InRegister(regNumber reg, unsigned offset, unsigned size); static ABIPassingSegment OnStack(unsigned stackOffset, unsigned offset, unsigned size); diff --git a/src/runtime/src/coreclr/jit/codegenloongarch64.cpp b/src/runtime/src/coreclr/jit/codegenloongarch64.cpp index 92474241b3e..bb6c8ab511c 100644 --- a/src/runtime/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/runtime/src/coreclr/jit/codegenloongarch64.cpp @@ -5846,6 +5846,16 @@ void CodeGen::genCallInstruction(GenTreeCall* call) { params.retSize = emitTypeSize(pRetTypeDesc->GetReturnRegType(0)); params.secondRetSize = emitTypeSize(pRetTypeDesc->GetReturnRegType(1)); + + if (pRetTypeDesc->GetABIReturnReg(1, call->GetUnmanagedCallConv()) == REG_INTRET) + { + // If the second return register is REG_INTRET, then the first return is expected to be in a floating + // register. The emitter has hardcoded belief that params.retSize corresponds to REG_INTRET and + // secondRetSize to REG_INTRET_1, so fix up the situation here. + assert(!EA_IS_GCREF_OR_BYREF(params.retSize)); + params.retSize = params.secondRetSize; + params.secondRetSize = EA_UNKNOWN; + } } else { diff --git a/src/runtime/src/coreclr/jit/codegenriscv64.cpp b/src/runtime/src/coreclr/jit/codegenriscv64.cpp index 24c2672ebcc..d841fbb0768 100644 --- a/src/runtime/src/coreclr/jit/codegenriscv64.cpp +++ b/src/runtime/src/coreclr/jit/codegenriscv64.cpp @@ -5939,6 +5939,16 @@ void CodeGen::genCallInstruction(GenTreeCall* call) { params.retSize = emitTypeSize(pRetTypeDesc->GetReturnRegType(0)); params.secondRetSize = emitTypeSize(pRetTypeDesc->GetReturnRegType(1)); + + if (pRetTypeDesc->GetABIReturnReg(1, call->GetUnmanagedCallConv()) == REG_INTRET) + { + // If the second return register is REG_INTRET, then the first return is expected to be in a floating + // register. The emitter has hardcoded belief that params.retSize corresponds to REG_INTRET and + // secondRetSize to REG_INTRET_1, so fix up the situation here. + assert(!EA_IS_GCREF_OR_BYREF(params.retSize)); + params.retSize = params.secondRetSize; + params.secondRetSize = EA_UNKNOWN; + } } else { diff --git a/src/runtime/src/coreclr/jit/codegenxarch.cpp b/src/runtime/src/coreclr/jit/codegenxarch.cpp index 9d11f754e13..3e40a1cba8f 100644 --- a/src/runtime/src/coreclr/jit/codegenxarch.cpp +++ b/src/runtime/src/coreclr/jit/codegenxarch.cpp @@ -9353,8 +9353,6 @@ void CodeGen::genAmd64EmitterUnitTestsApx() theEmitter->emitIns_R(INS_shl_1, EA_2BYTE, REG_R11, INS_OPTS_EVEX_nf); theEmitter->emitIns_R_I(INS_shl_N, EA_2BYTE, REG_R11, 7, INS_OPTS_EVEX_nf); theEmitter->emitIns_R_I(INS_shl_N, EA_2BYTE, REG_R11, 7, INS_OPTS_EVEX_nf); - theEmitter->emitIns_R_I(INS_rcr_N, EA_2BYTE, REG_R11, 7, INS_OPTS_EVEX_nf); - theEmitter->emitIns_R_I(INS_rcl_N, EA_2BYTE, REG_R11, 7, INS_OPTS_EVEX_nf); theEmitter->emitIns_R_R(INS_imul, EA_4BYTE, REG_R12, REG_R11, INS_OPTS_EVEX_nf); theEmitter->emitIns_R_S(INS_imul, EA_4BYTE, REG_R12, 0, 1, INS_OPTS_EVEX_nf); diff --git a/src/runtime/src/coreclr/jit/compiler.h b/src/runtime/src/coreclr/jit/compiler.h index 70edd4d4ae2..24d805d4149 100644 --- a/src/runtime/src/coreclr/jit/compiler.h +++ b/src/runtime/src/coreclr/jit/compiler.h @@ -6161,8 +6161,6 @@ class Compiler PhaseStatus fgHeadTailMerge(bool early); bool fgHeadMerge(BasicBlock* block, bool early); bool fgTryOneHeadMerge(BasicBlock* block, bool early); - template - bool gtTreeContainsCall(GenTree* tree, Predicate pred); bool gtTreeContainsTailCall(GenTree* tree); bool gtTreeContainsAsyncCall(GenTree* tree); bool fgCanMoveFirstStatementIntoPred(bool early, Statement* firstStmt, BasicBlock* pred); @@ -6479,6 +6477,7 @@ class Compiler bool fgPgoSynthesized; bool fgPgoDynamic; bool fgPgoConsistent; + bool fgPgoSingleEdge = false; #ifdef DEBUG bool fgPgoDeferredInconsistency; @@ -6914,6 +6913,9 @@ class Compiler bool gtIsTypeHandleToRuntimeTypeHelper(GenTreeCall* call); bool gtIsTypeHandleToRuntimeTypeHandleHelper(GenTreeCall* call, CorInfoHelpFunc* pHelper = nullptr); + template + GenTree* gtFindNodeInTree(GenTree* tree, Predicate predicate); + bool gtTreeContainsOper(GenTree* tree, genTreeOps op); ExceptionSetFlags gtCollectExceptions(GenTree* tree); @@ -11634,6 +11636,7 @@ class Compiler #endif // defined(UNIX_AMD64_ABI) bool fgTryMorphStructArg(CallArg* arg); + bool FieldsMatchAbi(LclVarDsc* varDsc, const ABIPassingInformation& abiInfo); bool killGCRefs(GenTree* tree); diff --git a/src/runtime/src/coreclr/jit/compiler.hpp b/src/runtime/src/coreclr/jit/compiler.hpp index 4ea69573670..d086f69ccfb 100644 --- a/src/runtime/src/coreclr/jit/compiler.hpp +++ b/src/runtime/src/coreclr/jit/compiler.hpp @@ -2199,6 +2199,74 @@ inline bool GenTree::gtOverflowEx() const return OperMayOverflow() && gtOverflow(); } +//------------------------------------------------------------------------ +// gtFindNodeInTree: +// Check if a tree contains a node matching the specified predicate. Descend +// only into subtrees with the specified flags set on them (can be GTF_EMPTY +// to descend into all nodes). +// +// Type parameters: +// RequiredFlagsToDescendIntoNode - Flags that must be set on the node to +// descend into it (GTF_EMPTY to descend into all nodes) +// Predicate - Type of the predicate (GenTree* -> bool) +// +// Parameters: +// tree - The tree +// pred - Predicate that the call must match +// +// Returns: +// Node matching the predicate, or nullptr if no such node was found. +// +template +GenTree* Compiler::gtFindNodeInTree(GenTree* tree, Predicate pred) +{ + struct FindNodeVisitor : GenTreeVisitor + { + private: + Predicate& m_pred; + + public: + GenTree* Result = nullptr; + + enum + { + DoPreOrder = true + }; + + FindNodeVisitor(Compiler* comp, Predicate& pred) + : GenTreeVisitor(comp) + , m_pred(pred) + { + } + + fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) + { + GenTree* node = *use; + if ((node->gtFlags & RequiredFlagsToDescendIntoNode) != RequiredFlagsToDescendIntoNode) + { + return WALK_SKIP_SUBTREES; + } + + if (m_pred(node)) + { + Result = node; + return WALK_ABORT; + } + + return WALK_CONTINUE; + } + }; + + if ((tree->gtFlags & RequiredFlagsToDescendIntoNode) != RequiredFlagsToDescendIntoNode) + { + return nullptr; + } + + FindNodeVisitor findNode(this, pred); + findNode.WalkTree(&tree, nullptr); + return findNode.Result; +} + /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX diff --git a/src/runtime/src/coreclr/jit/fgopt.cpp b/src/runtime/src/coreclr/jit/fgopt.cpp index 7dcfaee2703..fcc4f5db433 100644 --- a/src/runtime/src/coreclr/jit/fgopt.cpp +++ b/src/runtime/src/coreclr/jit/fgopt.cpp @@ -5604,63 +5604,6 @@ bool Compiler::fgHeadMerge(BasicBlock* block, bool early) return madeChanges; } -//------------------------------------------------------------------------ -// gtTreeContainsCall: -// Check if a tree contains a call node matching the given predicate. -// -// Parameters: -// tree - The tree -// pred - Predicate that the call must match -// -// Returns: -// True if a call node matching the predicate was found, false otherwise. -// -template -bool Compiler::gtTreeContainsCall(GenTree* tree, Predicate pred) -{ - struct HasCallVisitor : GenTreeVisitor - { - private: - Predicate& m_pred; - - public: - enum - { - DoPreOrder = true - }; - - HasCallVisitor(Compiler* comp, Predicate& pred) - : GenTreeVisitor(comp) - , m_pred(pred) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - GenTree* node = *use; - if ((node->gtFlags & GTF_CALL) == 0) - { - return WALK_SKIP_SUBTREES; - } - - if (node->IsCall() && m_pred(node->AsCall())) - { - return WALK_ABORT; - } - - return WALK_CONTINUE; - } - }; - - if ((tree->gtFlags & GTF_CALL) == 0) - { - return false; - } - - HasCallVisitor hasCall(this, pred); - return hasCall.WalkTree(&tree, nullptr) == WALK_ABORT; -} - //------------------------------------------------------------------------ // gtTreeContainsTailCall: Check if a tree contains any tail call or tail call // candidate. @@ -5676,9 +5619,11 @@ bool Compiler::gtTreeContainsCall(GenTree* tree, Predicate pred) // bool Compiler::gtTreeContainsTailCall(GenTree* tree) { - return gtTreeContainsCall(tree, [](GenTreeCall* call) { - return call->CanTailCall() || call->IsTailCall(); - }); + auto isTailCall = [](GenTree* tree) { + return tree->IsCall() && (tree->AsCall()->CanTailCall() || tree->AsCall()->IsTailCall()); + }; + + return gtFindNodeInTree(tree, isTailCall) != nullptr; } //------------------------------------------------------------------------ @@ -5697,9 +5642,11 @@ bool Compiler::gtTreeContainsAsyncCall(GenTree* tree) return false; } - return gtTreeContainsCall(tree, [](GenTreeCall* call) { - return call->IsAsync(); - }); + auto isAsyncCall = [](GenTree* tree) { + return tree->IsCall() && tree->AsCall()->IsAsync(); + }; + + return gtFindNodeInTree(tree, isAsyncCall) != nullptr; } //------------------------------------------------------------------------ diff --git a/src/runtime/src/coreclr/jit/fgprofile.cpp b/src/runtime/src/coreclr/jit/fgprofile.cpp index ba6a17697b6..9dd8ad2c027 100644 --- a/src/runtime/src/coreclr/jit/fgprofile.cpp +++ b/src/runtime/src/coreclr/jit/fgprofile.cpp @@ -89,6 +89,11 @@ bool Compiler::fgHaveSufficientProfileWeights() case ICorJitInfo::PgoSource::Blend: return true; + case ICorJitInfo::PgoSource::Synthesis: + // Single-edge methods always have sufficient profile data. + // Assuming we don't synthesize value and class profile data (which we don't currently). + return fgPgoSingleEdge; + case ICorJitInfo::PgoSource::Static: { // We sometimes call this very early, eg evaluating the prejit root. @@ -134,6 +139,12 @@ bool Compiler::fgHaveTrustedProfileWeights() case ICorJitInfo::PgoSource::Blend: case ICorJitInfo::PgoSource::Text: return true; + + case ICorJitInfo::PgoSource::Synthesis: + // Single-edge methods with synthetic profile are trustful. + // Assuming we don't synthesize value and class profile data (which we don't currently). + return fgPgoSingleEdge; + default: return false; } diff --git a/src/runtime/src/coreclr/jit/fgprofilesynthesis.cpp b/src/runtime/src/coreclr/jit/fgprofilesynthesis.cpp index fd9caa31eec..63fd3fd489d 100644 --- a/src/runtime/src/coreclr/jit/fgprofilesynthesis.cpp +++ b/src/runtime/src/coreclr/jit/fgprofilesynthesis.cpp @@ -148,6 +148,17 @@ void ProfileSynthesis::Run(ProfileSynthesisOption option) m_comp->fgPgoSynthesized = true; m_comp->fgPgoConsistent = !m_approximate; + // A simple check whether the current method has more than one edge. + m_comp->fgPgoSingleEdge = true; + for (BasicBlock* const block : m_comp->Blocks()) + { + if (block->NumSucc() > 1) + { + m_comp->fgPgoSingleEdge = false; + break; + } + } + m_comp->Metrics.ProfileSynthesizedBlendedOrRepaired++; if (m_approximate) diff --git a/src/runtime/src/coreclr/jit/forwardsub.cpp b/src/runtime/src/coreclr/jit/forwardsub.cpp index e22676e64f7..93d6d5308c3 100644 --- a/src/runtime/src/coreclr/jit/forwardsub.cpp +++ b/src/runtime/src/coreclr/jit/forwardsub.cpp @@ -498,20 +498,11 @@ bool Compiler::fgForwardSubStatement(Statement* stmt) // GenTree* fwdSubNode = defNode->AsLclVarCommon()->Data(); - // Can't substitute GT_CATCH_ARG. - // Can't substitute GT_LCLHEAP. + // Can't substitute GT_CATCH_ARG, GT_LCLHEAP or GT_ASYNC_CONTINUATION. // if (fwdSubNode->OperIs(GT_CATCH_ARG, GT_LCLHEAP, GT_ASYNC_CONTINUATION)) { - JITDUMP(" tree to sub is catch arg, or lcl heap\n"); - return false; - } - - // Don't substitute a no return call (trips up morph in some cases). - // - if (fwdSubNode->IsCall() && fwdSubNode->AsCall()->IsNoReturn()) - { - JITDUMP(" tree to sub is a 'no return' call\n"); + JITDUMP(" tree to sub is %s\n", GenTree::OpName(fwdSubNode->OperGet())); return false; } diff --git a/src/runtime/src/coreclr/jit/gentree.cpp b/src/runtime/src/coreclr/jit/gentree.cpp index 241f741eb1b..0baf062913b 100644 --- a/src/runtime/src/coreclr/jit/gentree.cpp +++ b/src/runtime/src/coreclr/jit/gentree.cpp @@ -17716,35 +17716,10 @@ bool Compiler::gtIsTypeHandleToRuntimeTypeHandleHelper(GenTreeCall* call, CorInf // bool Compiler::gtTreeContainsOper(GenTree* tree, genTreeOps oper) { - class Visitor final : public GenTreeVisitor - { - genTreeOps m_oper; - - public: - Visitor(Compiler* comp, genTreeOps oper) - : GenTreeVisitor(comp) - , m_oper(oper) - { - } - - enum - { - DoPreOrder = true, - }; - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - if ((*use)->OperIs(m_oper)) - { - return WALK_ABORT; - } - - return WALK_CONTINUE; - } + auto hasOper = [oper](GenTree* tree) { + return tree->OperGet() == oper; }; - - Visitor visitor(this, oper); - return visitor.WalkTree(&tree, nullptr) == WALK_ABORT; + return gtFindNodeInTree(tree, hasOper) != nullptr; } //------------------------------------------------------------------------ diff --git a/src/runtime/src/coreclr/jit/gentree.h b/src/runtime/src/coreclr/jit/gentree.h index c6efe050d7e..17f46648c2f 100644 --- a/src/runtime/src/coreclr/jit/gentree.h +++ b/src/runtime/src/coreclr/jit/gentree.h @@ -2735,16 +2735,10 @@ struct GenTreeFieldList : public GenTree class UseList { - Use* m_head; - Use* m_tail; + Use* m_head = nullptr; + Use* m_tail = nullptr; public: - UseList() - : m_head(nullptr) - , m_tail(nullptr) - { - } - Use* GetHead() const { return m_head; @@ -2802,6 +2796,12 @@ struct GenTreeFieldList : public GenTree } } + void Clear() + { + m_head = nullptr; + m_tail = nullptr; + } + bool IsSorted() const { unsigned offset = 0; diff --git a/src/runtime/src/coreclr/jit/gschecks.cpp b/src/runtime/src/coreclr/jit/gschecks.cpp index a47c62e9f39..8936ffd8a3b 100644 --- a/src/runtime/src/coreclr/jit/gschecks.cpp +++ b/src/runtime/src/coreclr/jit/gschecks.cpp @@ -537,37 +537,11 @@ void Compiler::gsParamsToShadows() // inserting reverse pinvoke transitions way too early in the // JIT. - struct HasReversePInvokeEnterVisitor : GenTreeVisitor - { - enum - { - DoPreOrder = true, - }; - - HasReversePInvokeEnterVisitor(Compiler* comp) - : GenTreeVisitor(comp) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - if (((*use)->gtFlags & GTF_CALL) == 0) - { - return fgWalkResult::WALK_SKIP_SUBTREES; - } - - if ((*use)->IsHelperCall(m_compiler, CORINFO_HELP_JIT_REVERSE_PINVOKE_ENTER) || - (*use)->IsHelperCall(m_compiler, CORINFO_HELP_JIT_REVERSE_PINVOKE_ENTER_TRACK_TRANSITIONS)) - { - return fgWalkResult::WALK_ABORT; - } - - return fgWalkResult::WALK_CONTINUE; - } + auto isReversePInvoke = [=](GenTree* tree) { + return tree->IsHelperCall(this, CORINFO_HELP_JIT_REVERSE_PINVOKE_ENTER) || + tree->IsHelperCall(this, CORINFO_HELP_JIT_REVERSE_PINVOKE_ENTER_TRACK_TRANSITIONS); }; - HasReversePInvokeEnterVisitor checker(this); - Statement* reversePInvokeStmt = nullptr; for (Statement* const stmt : fgFirstBB->Statements()) { @@ -575,7 +549,7 @@ void Compiler::gsParamsToShadows() // at the point before we insert the shadow copy statement. assert(!gtHasRef(stmt->GetRootNode(), lclNum)); - if (checker.WalkTree(stmt->GetRootNodePointer(), nullptr) == fgWalkResult::WALK_ABORT) + if (gtFindNodeInTree(stmt->GetRootNode(), isReversePInvoke) != nullptr) { reversePInvokeStmt = stmt; break; diff --git a/src/runtime/src/coreclr/jit/instrsxarch.h b/src/runtime/src/coreclr/jit/instrsxarch.h index 32fde631c70..ce3c34e2052 100644 --- a/src/runtime/src/coreclr/jit/instrsxarch.h +++ b/src/runtime/src/coreclr/jit/instrsxarch.h @@ -1029,12 +1029,12 @@ INST3(ror, "ror", IUM_RW, 0x0008D2, BAD_CODE, INST3(ror_1, "ror", IUM_RW, 0x0008D0, 0x0008D0, 0x0008D0, INS_TT_NONE, Writes_OF | Writes_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) INST3(ror_N, "ror", IUM_RW, 0x0008C0, 0x0008C0, BAD_CODE, INS_TT_NONE, Undefined_OF | Writes_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) -INST3(rcl, "rcl", IUM_RW, 0x0010D2, BAD_CODE, 0x0010D2, INS_TT_NONE, Undefined_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) -INST3(rcl_1, "rcl", IUM_RW, 0x0010D0, 0x0010D0, 0x0010D0, INS_TT_NONE, Writes_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) -INST3(rcl_N, "rcl", IUM_RW, 0x0010C0, 0x0010C0, BAD_CODE, INS_TT_NONE, Undefined_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) -INST3(rcr, "rcr", IUM_RW, 0x0018D2, BAD_CODE, 0x0018D2, INS_TT_NONE, Undefined_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) -INST3(rcr_1, "rcr", IUM_RW, 0x0018D0, 0x0018D0, 0x0018D0, INS_TT_NONE, Writes_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) -INST3(rcr_N, "rcr", IUM_RW, 0x0018C0, 0x0018C0, BAD_CODE, INS_TT_NONE, Undefined_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) +INST3(rcl, "rcl", IUM_RW, 0x0010D2, BAD_CODE, 0x0010D2, INS_TT_NONE, Undefined_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD) +INST3(rcl_1, "rcl", IUM_RW, 0x0010D0, 0x0010D0, 0x0010D0, INS_TT_NONE, Writes_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD) +INST3(rcl_N, "rcl", IUM_RW, 0x0010C0, 0x0010C0, BAD_CODE, INS_TT_NONE, Undefined_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD) +INST3(rcr, "rcr", IUM_RW, 0x0018D2, BAD_CODE, 0x0018D2, INS_TT_NONE, Undefined_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD) +INST3(rcr_1, "rcr", IUM_RW, 0x0018D0, 0x0018D0, 0x0018D0, INS_TT_NONE, Writes_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD) +INST3(rcr_N, "rcr", IUM_RW, 0x0018C0, 0x0018C0, BAD_CODE, INS_TT_NONE, Undefined_OF | Writes_CF | Reads_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD) INST3(shl, "shl", IUM_RW, 0x0020D2, BAD_CODE, 0x0020D2, INS_TT_NONE, Undefined_OF | Writes_SF | Writes_ZF | Undefined_AF | Writes_PF | Writes_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) INST3(shl_1, "shl", IUM_RW, 0x0020D0, 0x0020D0, 0x0020D0, INS_TT_NONE, Writes_OF | Writes_SF | Writes_ZF | Undefined_AF | Writes_PF | Writes_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) INST3(shl_N, "shl", IUM_RW, 0x0020C0, 0x0020C0, BAD_CODE, INS_TT_NONE, Undefined_OF | Writes_SF | Writes_ZF | Undefined_AF | Writes_PF | Writes_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF) diff --git a/src/runtime/src/coreclr/jit/lower.cpp b/src/runtime/src/coreclr/jit/lower.cpp index 5b13b649435..f961a27d858 100644 --- a/src/runtime/src/coreclr/jit/lower.cpp +++ b/src/runtime/src/coreclr/jit/lower.cpp @@ -1725,20 +1725,22 @@ void Lowering::LowerArg(GenTreeCall* call, CallArg* callArg) { if (abiInfo.HasAnyRegisterSegment()) { -#if FEATURE_MULTIREG_ARGS - if ((abiInfo.NumSegments > 1) && arg->OperIs(GT_FIELD_LIST)) + if (arg->OperIs(GT_FIELD_LIST) || (abiInfo.NumSegments > 1)) { - unsigned int regIndex = 0; - for (GenTreeFieldList::Use& use : arg->AsFieldList()->Uses()) + if (!arg->OperIs(GT_FIELD_LIST)) { - const ABIPassingSegment& segment = abiInfo.Segment(regIndex); - InsertPutArgReg(&use.NodeRef(), segment); - - regIndex++; + // Primitive arg, but the ABI requires it to be split into + // registers. Insert the field list here. + GenTreeFieldList* fieldList = comp->gtNewFieldList(); + fieldList->AddFieldLIR(comp, arg, 0, genActualType(arg->TypeGet())); + BlockRange().InsertAfter(arg, fieldList); + arg = *ppArg = fieldList; } + + LowerArgFieldList(callArg, arg->AsFieldList()); + arg = *ppArg; } else -#endif // FEATURE_MULTIREG_ARGS { assert(abiInfo.HasExactlyOneRegisterSegment()); InsertPutArgReg(ppArg, abiInfo.Segment(0)); @@ -4809,6 +4811,18 @@ void Lowering::LowerRet(GenTreeOp* ret) ContainCheckRet(ret); } +struct LowerFieldListRegisterInfo +{ + unsigned Offset; + var_types RegType; + + LowerFieldListRegisterInfo(unsigned offset, var_types regType) + : Offset(offset) + , RegType(regType) + { + } +}; + //---------------------------------------------------------------------------------------------- // LowerRetFieldList: // Lower a returned FIELD_LIST node. @@ -4822,21 +4836,18 @@ void Lowering::LowerRetFieldList(GenTreeOp* ret, GenTreeFieldList* fieldList) const ReturnTypeDesc& retDesc = comp->compRetTypeDesc; unsigned numRegs = retDesc.GetReturnRegCount(); - bool isCompatible = IsFieldListCompatibleWithReturn(fieldList); + auto getRegInfo = [=, &retDesc](unsigned regIndex) { + unsigned offset = retDesc.GetReturnFieldOffset(regIndex); + var_types regType = genActualType(retDesc.GetReturnRegType(regIndex)); + return LowerFieldListRegisterInfo(offset, regType); + }; + + bool isCompatible = IsFieldListCompatibleWithRegisters(fieldList, numRegs, getRegInfo); if (!isCompatible) { - JITDUMP("Spilling field list [%06u] to stack\n", Compiler::dspTreeID(fieldList)); - unsigned lclNum = comp->lvaGrabTemp(true DEBUGARG("Spilled local for return value")); + unsigned lclNum = + StoreFieldListToNewLocal(comp->typGetObjLayout(comp->info.compMethodInfo->args.retTypeClass), fieldList); LclVarDsc* varDsc = comp->lvaGetDesc(lclNum); - comp->lvaSetStruct(lclNum, comp->info.compMethodInfo->args.retTypeClass, false); - comp->lvaSetVarDoNotEnregister(lclNum DEBUGARG(DoNotEnregisterReason::BlockOpRet)); - - for (GenTreeFieldList::Use& use : fieldList->Uses()) - { - GenTree* store = comp->gtNewStoreLclFldNode(lclNum, use.GetType(), use.GetOffset(), use.GetNode()); - BlockRange().InsertAfter(use.GetNode(), store); - LowerNode(store); - } GenTree* retValue = comp->gtNewLclvNode(lclNum, varDsc->TypeGet()); ret->SetReturnValue(retValue); @@ -4859,7 +4870,89 @@ void Lowering::LowerRetFieldList(GenTreeOp* ret, GenTreeFieldList* fieldList) return; } - LowerFieldListToFieldListOfRegisters(fieldList); + LowerFieldListToFieldListOfRegisters(fieldList, numRegs, getRegInfo); +} + +//---------------------------------------------------------------------------------------------- +// StoreFieldListToNewLocal: +// Create a new local with the specified layout and store the specified +// fields of the specified FIELD_LIST into it. +// +// Arguments: +// layout - Layout of the new local +// fieldList - Fields to store to it +// +// Returns: +// Var number of new local. +// +unsigned Lowering::StoreFieldListToNewLocal(ClassLayout* layout, GenTreeFieldList* fieldList) +{ + JITDUMP("Spilling field list [%06u] to stack\n", Compiler::dspTreeID(fieldList)); + unsigned lclNum = comp->lvaGrabTemp(true DEBUGARG("Spilled local for field list")); + LclVarDsc* varDsc = comp->lvaGetDesc(lclNum); + comp->lvaSetStruct(lclNum, layout, false); + comp->lvaSetVarDoNotEnregister(lclNum DEBUGARG(DoNotEnregisterReason::LocalField)); + + for (GenTreeFieldList::Use& use : fieldList->Uses()) + { + GenTree* store = comp->gtNewStoreLclFldNode(lclNum, use.GetType(), use.GetOffset(), use.GetNode()); + BlockRange().InsertAfter(use.GetNode(), store); + LowerNode(store); + } + + return lclNum; +} + +//---------------------------------------------------------------------------------------------- +// LowerArgFieldList: +// Lower an argument FIELD_LIST node. +// +// Arguments: +// arg - The argument +// fieldList - The FIELD_LIST node +// +void Lowering::LowerArgFieldList(CallArg* arg, GenTreeFieldList* fieldList) +{ + assert(!arg->AbiInfo.HasAnyStackSegment()); + + auto getRegInfo = [=](unsigned regIndex) { + const ABIPassingSegment& seg = arg->AbiInfo.Segment(regIndex); + return LowerFieldListRegisterInfo(seg.Offset, seg.GetRegisterType()); + }; + + bool isCompatible = IsFieldListCompatibleWithRegisters(fieldList, arg->AbiInfo.NumSegments, getRegInfo); + if (!isCompatible) + { + ClassLayout* layout = comp->typGetObjLayout(arg->GetSignatureClassHandle()); + unsigned lclNum = StoreFieldListToNewLocal(layout, fieldList); + fieldList->Uses().Clear(); + for (const ABIPassingSegment& seg : arg->AbiInfo.Segments()) + { + GenTreeLclFld* fld = comp->gtNewLclFldNode(lclNum, seg.GetRegisterType(layout), seg.Offset); + fieldList->AddFieldLIR(comp, fld, seg.Offset, fld->TypeGet()); + BlockRange().InsertBefore(fieldList, fld); + } + } + else + { + LowerFieldListToFieldListOfRegisters(fieldList, arg->AbiInfo.NumSegments, getRegInfo); + } + + GenTreeFieldList::Use* field = fieldList->Uses().GetHead(); + for (const ABIPassingSegment& seg : arg->AbiInfo.Segments()) + { + assert((field != nullptr) && "Ran out of fields while inserting PUTARG_REG"); + InsertPutArgReg(&field->NodeRef(), seg); + field = field->GetNext(); + } + + assert((field == nullptr) && "Missed fields while inserting PUTARG_REG"); + + arg->NodeRef() = fieldList->SoleFieldOrThis(); + if (arg->GetNode() != fieldList) + { + BlockRange().Remove(fieldList); + } } //---------------------------------------------------------------------------------------------- @@ -4874,21 +4967,29 @@ void Lowering::LowerRetFieldList(GenTreeOp* ret, GenTreeFieldList* fieldList) // True if the fields of the FIELD_LIST are all direct insertions into the // return registers. // -bool Lowering::IsFieldListCompatibleWithReturn(GenTreeFieldList* fieldList) +template +bool Lowering::IsFieldListCompatibleWithRegisters(GenTreeFieldList* fieldList, + unsigned numRegs, + GetRegisterInfoFunc getRegInfo) { - JITDUMP("Checking if field list [%06u] is compatible with return ABI: ", Compiler::dspTreeID(fieldList)); - const ReturnTypeDesc& retDesc = comp->compRetTypeDesc; - unsigned numRetRegs = retDesc.GetReturnRegCount(); + JITDUMP("Checking if field list [%06u] is compatible with registers: ", Compiler::dspTreeID(fieldList)); GenTreeFieldList::Use* use = fieldList->Uses().GetHead(); - for (unsigned i = 0; i < numRetRegs; i++) + for (unsigned i = 0; i < numRegs; i++) { - unsigned regStart = retDesc.GetReturnFieldOffset(i); - var_types regType = retDesc.GetReturnRegType(i); - unsigned regEnd = regStart + genTypeSize(regType); + LowerFieldListRegisterInfo regInfo = getRegInfo(i); + unsigned regStart = regInfo.Offset; + var_types regType = regInfo.RegType; + unsigned regEnd = regStart + genTypeSize(regType); + + if ((i == numRegs - 1) && !varTypeUsesFloatReg(regType)) + { + // Allow tail end to pass undefined bits into the register + regEnd = regStart + REGSIZE_BYTES; + } // TODO-CQ: Could just create a 0 for this. - if (use == nullptr) + if ((use == nullptr) || (use->GetOffset() >= regEnd)) { JITDUMP("it is not; register %u has no corresponding field\n", i); return false; @@ -4949,19 +5050,26 @@ bool Lowering::IsFieldListCompatibleWithReturn(GenTreeFieldList* fieldList) // Arguments: // fieldList - The field list // -void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList) +template +void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList, + unsigned numRegs, + GetRegisterInfoFunc getRegInfo) { - const ReturnTypeDesc& retDesc = comp->compRetTypeDesc; - unsigned numRegs = retDesc.GetReturnRegCount(); - GenTreeFieldList::Use* use = fieldList->Uses().GetHead(); assert(fieldList->Uses().IsSorted()); for (unsigned i = 0; i < numRegs; i++) { - unsigned regStart = retDesc.GetReturnFieldOffset(i); - var_types regType = genActualType(retDesc.GetReturnRegType(i)); - unsigned regEnd = regStart + genTypeSize(regType); + LowerFieldListRegisterInfo regInfo = getRegInfo(i); + unsigned regStart = regInfo.Offset; + var_types regType = regInfo.RegType; + unsigned regEnd = regStart + genTypeSize(regType); + + if ((i == numRegs - 1) && !varTypeUsesFloatReg(regType)) + { + // Allow tail end to pass undefined bits into the register + regEnd = regStart + REGSIZE_BYTES; + } GenTreeFieldList::Use* regEntry = use; @@ -5001,7 +5109,7 @@ void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList) } // If this is a float -> int insertion, then we need the bitcast now. - if (varTypeUsesFloatReg(value) && varTypeUsesIntReg(regType)) + if (varTypeUsesFloatReg(value) && varTypeUsesIntReg(regInfo.RegType)) { assert((genTypeSize(value) == 4) || (genTypeSize(value) == 8)); var_types castType = genTypeSize(value) == 4 ? TYP_INT : TYP_LONG; diff --git a/src/runtime/src/coreclr/jit/lower.h b/src/runtime/src/coreclr/jit/lower.h index 019a2f2b48a..75507bd6caa 100644 --- a/src/runtime/src/coreclr/jit/lower.h +++ b/src/runtime/src/coreclr/jit/lower.h @@ -187,10 +187,14 @@ class Lowering final : public Phase GenTree* LowerAsyncContinuation(GenTree* asyncCont); void LowerReturnSuspend(GenTree* retSuspend); void LowerRetFieldList(GenTreeOp* ret, GenTreeFieldList* fieldList); - bool IsFieldListCompatibleWithReturn(GenTreeFieldList* fieldList); - void LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList); - void LowerCallStruct(GenTreeCall* call); - void LowerStoreSingleRegCallStruct(GenTreeBlk* store); + unsigned StoreFieldListToNewLocal(ClassLayout* layout, GenTreeFieldList* fieldList); + void LowerArgFieldList(CallArg* arg, GenTreeFieldList* fieldList); + template + bool IsFieldListCompatibleWithRegisters(GenTreeFieldList* fieldList, unsigned numRegs, GetRegisterInfoFunc func); + template + void LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList, unsigned numRegs, GetRegisterInfoFunc func); + void LowerCallStruct(GenTreeCall* call); + void LowerStoreSingleRegCallStruct(GenTreeBlk* store); #if !defined(WINDOWS_AMD64_ABI) GenTreeLclVar* SpillStructCallResult(GenTreeCall* call) const; #endif // WINDOWS_AMD64_ABI diff --git a/src/runtime/src/coreclr/jit/morph.cpp b/src/runtime/src/coreclr/jit/morph.cpp index 90c1a5ae7f4..6b09056e0e3 100644 --- a/src/runtime/src/coreclr/jit/morph.cpp +++ b/src/runtime/src/coreclr/jit/morph.cpp @@ -2219,11 +2219,11 @@ GenTreeCall* Compiler::fgMorphArgs(GenTreeCall* call) // False if the argument cannot be put into a shape supported by the backend. // // Remarks: -// The backend requires register-passed arguments to be of FIELD_LIST shape -// with one primitive node for each register. If the argument is passed in -// only one register, then the FIELD_LIST should be directly replaced by its -// operand. For stack-passed arguments the backend supports struct-typed -// arguments directly. +// The backend requires register-passed arguments to be of FIELD_LIST shape. +// For split arguments it is additionally required that registers and stack +// slots have clean mappings to fields. +// For stack-passed arguments the backend supports struct-typed arguments +// directly. // bool Compiler::fgTryMorphStructArg(CallArg* arg) { @@ -2299,49 +2299,20 @@ bool Compiler::fgTryMorphStructArg(CallArg* arg) JITDUMP("Struct argument V%02u: ", lclNum); JITDUMPEXEC(arg->Dump(this)); - // Try to see if we can use the promoted fields to pass this argument. + // Try to see if we can and should use promoted fields to pass this + // argument. // - if (varDsc->lvPromoted && !varDsc->lvDoNotEnregister && - (varDsc->lvFieldCnt == arg->AbiInfo.CountRegsAndStackSlots())) + if (varDsc->lvPromoted && !varDsc->lvDoNotEnregister && (!isSplit || FieldsMatchAbi(varDsc, arg->AbiInfo))) { - bool fieldsMatch = true; - - for (const ABIPassingSegment& seg : arg->AbiInfo.Segments()) - { - if (seg.IsPassedInRegister()) - { - unsigned fieldLclNum = lvaGetFieldLocal(varDsc, seg.Offset); - if (fieldLclNum == BAD_VAR_NUM) - { - fieldsMatch = false; - break; - } - } - else - { - for (unsigned offset = 0; offset < seg.Size; offset += TARGET_POINTER_SIZE) - { - if (lvaGetFieldLocal(varDsc, seg.Offset + offset) == BAD_VAR_NUM) - { - fieldsMatch = false; - break; - } - } - } - } - - if (fieldsMatch) - { - newArg = fgMorphLclToFieldList(lclNode)->SoleFieldOrThis(); - newArg = fgMorphTree(newArg); - } + newArg = fgMorphLclToFieldList(lclNode)->SoleFieldOrThis(); + newArg = fgMorphTree(newArg); } } else if (argNode->OperIsFieldList()) { // We can already see a field list here if physical promotion created it. // Physical promotion will also create single-field field lists which - // the backend does not expect, so fix that here. + // not everything treats the same as a single node, so fix that here. newArg = argNode->AsFieldList()->SoleFieldOrThis(); if (newArg == argNode) { @@ -2353,20 +2324,7 @@ bool Compiler::fgTryMorphStructArg(CallArg* arg) // if (newArg == nullptr) { - bool isUseOfIndependentlyPromotedStruct = - argNode->OperIs(GT_LCL_VAR) && - (lvaGetPromotionType(argNode->AsLclVarCommon()->GetLclNum()) == PROMOTION_TYPE_INDEPENDENT); - if (isUseOfIndependentlyPromotedStruct) - { - if (arg->AbiInfo.HasExactlyOneRegisterSegment()) - { - // We already tried to use the fields above, but that failed. - // Here we prefer to create a copy to avoid DNER'ing the local. - // That turns out to be profitable mostly for single-register arguments. - return false; - } - } - else if (!argNode->TypeIs(TYP_STRUCT) && arg->AbiInfo.HasExactlyOneRegisterSegment()) + if (!argNode->TypeIs(TYP_STRUCT) && arg->AbiInfo.HasExactlyOneRegisterSegment()) { // This can be treated primitively. Leave it alone. return true; @@ -2544,6 +2502,51 @@ bool Compiler::fgTryMorphStructArg(CallArg* arg) return true; } +//----------------------------------------------------------------------------- +// FieldsMatchAbi: +// Check if the fields of a local map cleanly (in terms of offsets) to the +// specified ABI info. +// +// Arguments: +// varDsc - promoted local +// abiInfo - ABI information +// +// Returns: +// True if it does. In that case FIELD_LIST usage is allowed for split args +// by the backend. +// +bool Compiler::FieldsMatchAbi(LclVarDsc* varDsc, const ABIPassingInformation& abiInfo) +{ + if (varDsc->lvFieldCnt != abiInfo.CountRegsAndStackSlots()) + { + return false; + } + + for (const ABIPassingSegment& seg : abiInfo.Segments()) + { + if (seg.IsPassedInRegister()) + { + unsigned fieldLclNum = lvaGetFieldLocal(varDsc, seg.Offset); + if (fieldLclNum == BAD_VAR_NUM) + { + return false; + } + } + else + { + for (unsigned offset = 0; offset < seg.Size; offset += TARGET_POINTER_SIZE) + { + if (lvaGetFieldLocal(varDsc, seg.Offset + offset) == BAD_VAR_NUM) + { + return false; + } + } + } + } + + return true; +} + //------------------------------------------------------------------------ // fgMorphLclToFieldList: Morph a GT_LCL_VAR node to a GT_FIELD_LIST of its promoted fields // diff --git a/src/runtime/src/coreclr/jit/optcse.cpp b/src/runtime/src/coreclr/jit/optcse.cpp index f96b38c5ee1..88813713f6a 100644 --- a/src/runtime/src/coreclr/jit/optcse.cpp +++ b/src/runtime/src/coreclr/jit/optcse.cpp @@ -298,8 +298,8 @@ bool Compiler::optCSE_canSwap(GenTree* op1, GenTree* op2) /* static */ bool Compiler::optCSEcostCmpEx::operator()(const CSEdsc* dsc1, const CSEdsc* dsc2) { - GenTree* exp1 = dsc1->csdTree; - GenTree* exp2 = dsc2->csdTree; + GenTree* exp1 = dsc1->csdTreeList.tslTree; + GenTree* exp2 = dsc2->csdTreeList.tslTree; auto expCost1 = exp1->GetCostEx(); auto expCost2 = exp2->GetCostEx(); @@ -334,8 +334,8 @@ bool Compiler::optCSEcostCmpEx::operator()(const CSEdsc* dsc1, const CSEdsc* dsc /* static */ bool Compiler::optCSEcostCmpSz::operator()(const CSEdsc* dsc1, const CSEdsc* dsc2) { - GenTree* exp1 = dsc1->csdTree; - GenTree* exp2 = dsc2->csdTree; + GenTree* exp1 = dsc1->csdTreeList.tslTree; + GenTree* exp2 = dsc2->csdTreeList.tslTree; auto expCost1 = exp1->GetCostSz(); auto expCost2 = exp2->GetCostSz(); @@ -434,7 +434,7 @@ void CSEdsc::ComputeNumLocals(Compiler* compiler) }; LocalCountingVisitor lcv(compiler); - lcv.WalkTree(&csdTree, nullptr); + lcv.WalkTree(&csdTreeList.tslTree, nullptr); numDistinctLocals = lcv.m_count; numLocalOccurrences = lcv.m_occurrences; @@ -615,99 +615,85 @@ unsigned Compiler::optValnumCSE_Index(GenTree* tree, Statement* stmt) for (hashDsc = optCSEhash[hval]; hashDsc; hashDsc = hashDsc->csdNextInBucket) { - if (hashDsc->csdHashKey == key) + if (hashDsc->csdHashKey != key) { - // Check for mismatched types on GT_CNS_INT nodes - if ((tree->OperGet() == GT_CNS_INT) && (tree->TypeGet() != hashDsc->csdTree->TypeGet())) - { - continue; - } + continue; + } + + assert(hashDsc->csdTreeList.tslTree != nullptr); - treeStmtLst* newElem; + // Check for mismatched types on GT_CNS_INT nodes + if ((tree->OperGet() == GT_CNS_INT) && (tree->TypeGet() != hashDsc->csdTreeList.tslTree->TypeGet())) + { + continue; + } - // Have we started the list of matching nodes? + // Have we started the list of matching nodes? - if (hashDsc->csdTreeList == nullptr) + if (hashDsc->csdTreeList.tslNext == nullptr) + { + // This is the second time we see this value. Handle cases + // where the first value dominates the second one and we can + // already prove that the first one is _not_ going to be a + // valid def for the second one, due to the second one having + // more exceptions. This happens for example in code like + // CASTCLASS(x, y) where the "CASTCLASS" just adds exceptions + // on top of "x". In those cases it is always better to let the + // second value be the def. + // It also happens for GT_COMMA, but that one is special cased + // above; this handling is a less special-casey version of the + // GT_COMMA handling above. However, it is quite limited since + // it only handles the def/use being in the same block. + if (compCurBB == hashDsc->csdTreeList.tslBlock) { - // This is the second time we see this value. Handle cases - // where the first value dominates the second one and we can - // already prove that the first one is _not_ going to be a - // valid def for the second one, due to the second one having - // more exceptions. This happens for example in code like - // CASTCLASS(x, y) where the "CASTCLASS" just adds exceptions - // on top of "x". In those cases it is always better to let the - // second value be the def. - // It also happens for GT_COMMA, but that one is special cased - // above; this handling is a less special-casey version of the - // GT_COMMA handling above. However, it is quite limited since - // it only handles the def/use being in the same block. - if (compCurBB == hashDsc->csdBlock) + GenTree* prevTree = hashDsc->csdTreeList.tslTree; + ValueNum prevVnLib = prevTree->GetVN(VNK_Liberal); + if (prevVnLib != vnLib) { - GenTree* prevTree = hashDsc->csdTree; - ValueNum prevVnLib = prevTree->GetVN(VNK_Liberal); - if (prevVnLib != vnLib) + ValueNum prevExceptionSet = vnStore->VNExceptionSet(prevVnLib); + ValueNum curExceptionSet = vnStore->VNExceptionSet(vnLib); + if ((prevExceptionSet != curExceptionSet) && + vnStore->VNExcIsSubset(curExceptionSet, prevExceptionSet)) { - ValueNum prevExceptionSet = vnStore->VNExceptionSet(prevVnLib); - ValueNum curExceptionSet = vnStore->VNExceptionSet(vnLib); - if ((prevExceptionSet != curExceptionSet) && - vnStore->VNExcIsSubset(curExceptionSet, prevExceptionSet)) - { - JITDUMP("Skipping CSE candidate for tree [%06u]; tree [%06u] is a better candidate with " - "more exceptions\n", - prevTree->gtTreeID, tree->gtTreeID); - prevTree->gtCSEnum = 0; - hashDsc->csdStmt = stmt; - hashDsc->csdTree = tree; - tree->gtCSEnum = (signed char)hashDsc->csdIndex; - return hashDsc->csdIndex; - } + JITDUMP("Skipping CSE candidate for tree [%06u]; tree [%06u] is a better candidate with " + "more exceptions\n", + prevTree->gtTreeID, tree->gtTreeID); + prevTree->gtCSEnum = 0; + hashDsc->csdTreeList.tslStmt = stmt; + hashDsc->csdTreeList.tslTree = tree; + tree->gtCSEnum = (signed char)hashDsc->csdIndex; + return hashDsc->csdIndex; } } - - // Create the new element based upon the matching hashDsc element. - - newElem = new (this, CMK_TreeStatementList) treeStmtLst; - - newElem->tslTree = hashDsc->csdTree; - newElem->tslStmt = hashDsc->csdStmt; - newElem->tslBlock = hashDsc->csdBlock; - newElem->tslNext = nullptr; - - /* Start the list with the first CSE candidate recorded */ - - hashDsc->csdTreeList = newElem; - hashDsc->csdTreeLast = newElem; - - hashDsc->csdIsSharedConst = isSharedConst; } - noway_assert(hashDsc->csdTreeList); + hashDsc->csdIsSharedConst = isSharedConst; + } - /* Append this expression to the end of the list */ + // Append this expression to the end of the list - newElem = new (this, CMK_TreeStatementList) treeStmtLst; + treeStmtLst* newElem = new (this, CMK_TreeStatementList) treeStmtLst; - newElem->tslTree = tree; - newElem->tslStmt = stmt; - newElem->tslBlock = compCurBB; - newElem->tslNext = nullptr; + newElem->tslTree = tree; + newElem->tslStmt = stmt; + newElem->tslBlock = compCurBB; + newElem->tslNext = nullptr; - hashDsc->csdTreeLast->tslNext = newElem; - hashDsc->csdTreeLast = newElem; + hashDsc->csdTreeLast->tslNext = newElem; + hashDsc->csdTreeLast = newElem; - optDoCSE = true; // Found a duplicate CSE tree + optDoCSE = true; // Found a duplicate CSE tree - /* Have we assigned a CSE index? */ - if (hashDsc->csdIndex == 0) - { - newCSE = true; - break; - } - - assert(FitsIn(hashDsc->csdIndex)); - tree->gtCSEnum = ((signed char)hashDsc->csdIndex); - return hashDsc->csdIndex; + /* Have we assigned a CSE index? */ + if (hashDsc->csdIndex == 0) + { + newCSE = true; + break; } + + assert(FitsIn(hashDsc->csdIndex)); + tree->gtCSEnum = ((signed char)hashDsc->csdIndex); + return hashDsc->csdIndex; } if (!newCSE) @@ -763,10 +749,12 @@ unsigned Compiler::optValnumCSE_Index(GenTree* tree, Statement* stmt) hashDsc->defExcSetPromise = vnStore->VNForEmptyExcSet(); hashDsc->defExcSetCurrent = vnStore->VNForNull(); // uninit value - hashDsc->csdTree = tree; - hashDsc->csdStmt = stmt; - hashDsc->csdBlock = compCurBB; - hashDsc->csdTreeList = nullptr; + hashDsc->csdTreeList.tslTree = tree; + hashDsc->csdTreeList.tslStmt = stmt; + hashDsc->csdTreeList.tslBlock = compCurBB; + hashDsc->csdTreeList.tslNext = nullptr; + + hashDsc->csdTreeLast = &hashDsc->csdTreeList; /* Append the entry to the hash bucket */ @@ -801,11 +789,11 @@ unsigned Compiler::optValnumCSE_Index(GenTree* tree, Statement* stmt) hashDsc->csdIndex = CSEindex; /* Update the gtCSEnum field in the original tree */ - noway_assert(hashDsc->csdTreeList->tslTree->gtCSEnum == 0); + noway_assert(hashDsc->csdTreeList.tslTree->gtCSEnum == 0); assert(FitsIn(CSEindex)); - hashDsc->csdTreeList->tslTree->gtCSEnum = ((signed char)CSEindex); - noway_assert(((unsigned)hashDsc->csdTreeList->tslTree->gtCSEnum) == CSEindex); + hashDsc->csdTreeList.tslTree->gtCSEnum = ((signed char)CSEindex); + noway_assert(((unsigned)hashDsc->csdTreeList.tslTree->gtCSEnum) == CSEindex); tree->gtCSEnum = ((signed char)CSEindex); @@ -975,7 +963,7 @@ void Compiler::optValnumCSE_InitDataFlow() { CSEdsc* dsc = optCSEtab[inx]; unsigned CSEindex = dsc->csdIndex; - treeStmtLst* lst = dsc->csdTreeList; + treeStmtLst* lst = &dsc->csdTreeList; noway_assert(lst); while (lst != nullptr) @@ -1101,13 +1089,13 @@ void Compiler::optValnumCSE_SetUpAsyncByrefKills() CSEdsc* dsc = optCSEtab[inx - 1]; assert(dsc->csdIndex == inx); bool isByRef = false; - if (dsc->csdTree->TypeIs(TYP_BYREF)) + if (dsc->csdTreeList.tslTree->TypeIs(TYP_BYREF)) { isByRef = true; } - else if (dsc->csdTree->TypeIs(TYP_STRUCT)) + else if (dsc->csdTreeList.tslTree->TypeIs(TYP_STRUCT)) { - ClassLayout* layout = dsc->csdTree->GetLayout(this); + ClassLayout* layout = dsc->csdTreeList.tslTree->GetLayout(this); isByRef = layout->HasGCByRef(); } @@ -2528,14 +2516,14 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) return; } - const unsigned char costEx = cse->csdTree->GetCostEx(); + const unsigned char costEx = cse->csdTreeList.tslTree->GetCostEx(); const double deMinimis = 1e-3; const double deMinimusAdj = -log(deMinimis); features[0] = costEx; features[1] = deMinimusAdj + log(max(deMinimis, cse->csdUseWtCnt)); features[2] = deMinimusAdj + log(max(deMinimis, cse->csdDefWtCnt)); - features[3] = cse->csdTree->GetCostSz(); + features[3] = cse->csdTreeList.tslTree->GetCostSz(); features[4] = cse->csdUseCount; features[5] = cse->csdDefCount; @@ -2545,9 +2533,9 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) const bool isLiveAcrossCall = cse->csdLiveAcrossCall; features[6] = booleanScale * isLiveAcrossCall; - features[7] = booleanScale * varTypeUsesIntReg(cse->csdTree->TypeGet()); + features[7] = booleanScale * varTypeUsesIntReg(cse->csdTreeList.tslTree->TypeGet()); - const bool isConstant = cse->csdTree->OperIsConst(); + const bool isConstant = cse->csdTreeList.tslTree->OperIsConst(); const bool isSharedConstant = cse->csdIsSharedConst; features[8] = booleanScale * (isConstant & !isSharedConstant); @@ -2573,7 +2561,7 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) unsigned maxPostorderNum = 0; BasicBlock* minPostorderBlock = nullptr; BasicBlock* maxPostorderBlock = nullptr; - for (treeStmtLst* treeList = cse->csdTreeList; treeList != nullptr; treeList = treeList->tslNext) + for (treeStmtLst* treeList = &cse->csdTreeList; treeList != nullptr; treeList = treeList->tslNext) { BasicBlock* const treeBlock = treeList->tslBlock; unsigned postorderNum = treeBlock->bbPostorderNum; @@ -2602,12 +2590,12 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) // More // - features[17] = booleanScale * ((cse->csdTree->gtFlags & GTF_CALL) != 0); + features[17] = booleanScale * ((cse->csdTreeList.tslTree->gtFlags & GTF_CALL) != 0); features[18] = deMinimusAdj + log(max(deMinimis, cse->csdUseCount * cse->csdUseWtCnt)); features[19] = deMinimusAdj + log(max(deMinimis, cse->numLocalOccurrences * cse->csdUseWtCnt)); features[20] = booleanScale * ((double)(blockSpread) / numBBs); - const bool isContainable = cse->csdTree->OperIs(GT_ADD, GT_NOT, GT_MUL, GT_LSH); + const bool isContainable = cse->csdTreeList.tslTree->OperIs(GT_ADD, GT_NOT, GT_MUL, GT_LSH); features[21] = booleanScale * isContainable; features[22] = booleanScale * (isContainable && isLowCost); @@ -3172,7 +3160,7 @@ void CSE_HeuristicRLHook::GetFeatures(CSEdsc* cse, int* features) unsigned maxPostorderNum = 0; BasicBlock* minPostorderBlock = nullptr; BasicBlock* maxPostorderBlock = nullptr; - for (treeStmtLst* treeList = cse->csdTreeList; treeList != nullptr; treeList = treeList->tslNext) + for (treeStmtLst* treeList = &cse->csdTreeList; treeList != nullptr; treeList = treeList->tslNext) { BasicBlock* const treeBlock = treeList->tslBlock; unsigned postorderNum = treeBlock->bbPostorderNum; @@ -3232,13 +3220,13 @@ void CSE_HeuristicRLHook::GetFeatures(CSEdsc* cse, int* features) features[i++] = type; features[i++] = cse->IsViable() ? 1 : 0; features[i++] = cse->csdLiveAcrossCall ? 1 : 0; - features[i++] = cse->csdTree->OperIsConst() ? 1 : 0; + features[i++] = cse->csdTreeList.tslTree->OperIsConst() ? 1 : 0; features[i++] = cse->csdIsSharedConst ? 1 : 0; features[i++] = isMakeCse ? 1 : 0; - features[i++] = ((cse->csdTree->gtFlags & GTF_CALL) != 0) ? 1 : 0; - features[i++] = cse->csdTree->OperIs(GT_ADD, GT_NOT, GT_MUL, GT_LSH) ? 1 : 0; - features[i++] = cse->csdTree->GetCostEx(); - features[i++] = cse->csdTree->GetCostSz(); + features[i++] = ((cse->csdTreeList.tslTree->gtFlags & GTF_CALL) != 0) ? 1 : 0; + features[i++] = cse->csdTreeList.tslTree->OperIs(GT_ADD, GT_NOT, GT_MUL, GT_LSH) ? 1 : 0; + features[i++] = cse->csdTreeList.tslTree->GetCostEx(); + features[i++] = cse->csdTreeList.tslTree->GetCostSz(); features[i++] = cse->csdUseCount; features[i++] = cse->csdDefCount; features[i++] = (int)cse->csdUseWtCnt; @@ -4271,7 +4259,7 @@ void CSE_Heuristic::SortCandidates() for (unsigned cnt = 0; cnt < m_pCompiler->optCSECandidateCount; cnt++) { CSEdsc* dsc = sortTab[cnt]; - GenTree* expr = dsc->csdTree; + GenTree* expr = dsc->csdTreeList.tslTree; weight_t def; weight_t use; @@ -4281,13 +4269,13 @@ void CSE_Heuristic::SortCandidates() { def = dsc->csdDefCount; // def count use = dsc->csdUseCount; // use count (excluding the implicit uses at defs) - cost = dsc->csdTree->GetCostSz(); + cost = dsc->csdTreeList.tslTree->GetCostSz(); } else { def = dsc->csdDefWtCnt; // weighted def count use = dsc->csdUseWtCnt; // weighted use count (excluding the implicit uses at defs) - cost = dsc->csdTree->GetCostEx(); + cost = dsc->csdTreeList.tslTree->GetCostEx(); } if (!Compiler::Is_Shared_Const_CSE(dsc->csdHashKey)) @@ -4840,7 +4828,7 @@ void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) ValueNum bestVN = ValueNumStore::NoVN; bool bestIsDef = false; ssize_t bestConstValue = 0; - treeStmtLst* lst = dsc->csdTreeList; + treeStmtLst* lst = &dsc->csdTreeList; while (lst != nullptr) { @@ -4931,7 +4919,7 @@ void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) } else // !isSharedConst { - lst = dsc->csdTreeList; + lst = &dsc->csdTreeList; GenTree* firstTree = lst->tslTree; printf("In %s, CSE (oper = %s, type = %s) has differing VNs: ", m_pCompiler->info.compFullName, GenTree::OpName(firstTree->OperGet()), varTypeName(firstTree->TypeGet())); @@ -4956,7 +4944,7 @@ void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) ArrayStack defUses(m_pCompiler->getAllocator(CMK_CSE)); // First process the defs. - for (lst = dsc->csdTreeList; lst != nullptr; lst = lst->tslNext) + for (lst = &dsc->csdTreeList; lst != nullptr; lst = lst->tslNext) { GenTree* const exp = lst->tslTree; Statement* const stmt = lst->tslStmt; @@ -5065,7 +5053,7 @@ void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) } // Now process the actual uses. - for (lst = dsc->csdTreeList; lst != nullptr; lst = lst->tslNext) + for (lst = &dsc->csdTreeList; lst != nullptr; lst = lst->tslNext) { GenTree* const exp = lst->tslTree; Statement* const stmt = lst->tslStmt; diff --git a/src/runtime/src/coreclr/jit/optcse.h b/src/runtime/src/coreclr/jit/optcse.h index b6d9cda3b92..a6afa85c87c 100644 --- a/src/runtime/src/coreclr/jit/optcse.h +++ b/src/runtime/src/coreclr/jit/optcse.h @@ -362,11 +362,7 @@ struct CSEdsc weight_t csdDefWtCnt; // weighted def count weight_t csdUseWtCnt; // weighted use count (excluding the implicit uses at defs) - GenTree* csdTree; // treenode containing the 1st occurrence - Statement* csdStmt; // stmt containing the 1st occurrence - BasicBlock* csdBlock; // block containing the 1st occurrence - - treeStmtLst* csdTreeList; // list of matching tree nodes: head + treeStmtLst csdTreeList; // list of matching tree nodes: head treeStmtLst* csdTreeLast; // list of matching tree nodes: tail // The exception set that is now required for all defs of this CSE. @@ -500,7 +496,7 @@ class CSE_Candidate // TODO-CQ: With ValNum CSE's the Expr and its cost can vary. GenTree* Expr() { - return m_CseDsc->csdTree; + return m_CseDsc->csdTreeList.tslTree; } unsigned Cost() { diff --git a/src/runtime/src/coreclr/jit/optimizer.cpp b/src/runtime/src/coreclr/jit/optimizer.cpp index 61e264a7df2..e14a8724fab 100644 --- a/src/runtime/src/coreclr/jit/optimizer.cpp +++ b/src/runtime/src/coreclr/jit/optimizer.cpp @@ -3674,30 +3674,7 @@ void Compiler::optPerformHoistExpr(GenTree* origExpr, BasicBlock* exprBb, FlowGr preheader->CopyFlags(exprBb, BBF_COPY_PROPAGATE); - Statement* hoistStmt = gtNewStmt(hoist); - - // Simply append the statement at the end of the preHead's list. - Statement* firstStmt = preheader->firstStmt(); - if (firstStmt != nullptr) - { - /* append after last statement */ - - Statement* lastStmt = preheader->lastStmt(); - assert(lastStmt->GetNextStmt() == nullptr); - - lastStmt->SetNextStmt(hoistStmt); - hoistStmt->SetPrevStmt(lastStmt); - firstStmt->SetPrevStmt(hoistStmt); - } - else - { - /* Empty pre-header - store the single statement in the block */ - - preheader->bbStmtList = hoistStmt; - hoistStmt->SetPrevStmt(hoistStmt); - } - - hoistStmt->SetNextStmt(nullptr); + fgInsertStmtAtEnd(preheader, fgNewStmtFromTree(hoist)); #ifdef DEBUG if (verbose) @@ -3708,12 +3685,6 @@ void Compiler::optPerformHoistExpr(GenTree* origExpr, BasicBlock* exprBb, FlowGr } #endif - if (fgNodeThreading == NodeThreading::AllTrees) - { - gtSetStmtInfo(hoistStmt); - fgSetStmtSeq(hoistStmt); - } - #ifdef DEBUG if (m_nodeTestData != nullptr) { @@ -4304,15 +4275,20 @@ void Compiler::optRecordLoopMemoryDependence(GenTree* tree, BasicBlock* block, V } //------------------------------------------------------------------------ -// optCopyLoopMemoryDependence: record that tree's loop memory dependence +// optCopyLoopMemoryDependence: Recursively record that tree's loop memory dependence // is the same as some other tree. // // Arguments: // fromTree -- tree to copy dependence from // toTree -- tree in question // +// Remarks: +// This requires 'toTree' to be in its own statement +// void Compiler::optCopyLoopMemoryDependence(GenTree* fromTree, GenTree* toTree) { + assert(fromTree->OperGet() == toTree->OperGet()); + NodeToLoopMemoryBlockMap* const map = GetNodeToLoopMemoryBlockMap(); BasicBlock* mapBlock = nullptr; @@ -4320,6 +4296,20 @@ void Compiler::optCopyLoopMemoryDependence(GenTree* fromTree, GenTree* toTree) { map->Set(toTree, mapBlock); } + + GenTreeOperandIterator fromIterCur = fromTree->OperandsBegin(); + GenTreeOperandIterator fromIterEnd = fromTree->OperandsEnd(); + GenTreeOperandIterator toIterCur = toTree->OperandsBegin(); + GenTreeOperandIterator toIterEnd = toTree->OperandsEnd(); + + while (fromIterCur != fromIterEnd) + { + optCopyLoopMemoryDependence(*fromIterCur, *toIterCur); + ++fromIterCur; + ++toIterCur; + } + + assert(toIterCur == toIterEnd); } //------------------------------------------------------------------------ diff --git a/src/runtime/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj b/src/runtime/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj index 251d8bfa97e..18eed4a5d7d 100644 --- a/src/runtime/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj +++ b/src/runtime/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj @@ -559,30 +559,8 @@ - - - - - - src\System\Diagnostics\Eventing\NativeRuntimeEventSource.Generated.cs - - + + + - - - - - - <_PythonWarningParameter>-Wall - <_PythonWarningParameter Condition="'$(MSBuildTreatWarningsAsErrors)' == 'true'">$(_PythonWarningParameter) -Werror - <_EventingSourceFileDirectory>%(EventingSourceFile.RootDir)%(EventingSourceFile.Directory) - <_EventingSourceFileDirectory Condition="HasTrailingSlash('$(_EventingSourceFileDirectory)')">$(_EventingSourceFileDirectory.TrimEnd('\')) - - - - - - - - diff --git a/src/runtime/src/coreclr/pal/src/exception/signal.cpp b/src/runtime/src/coreclr/pal/src/exception/signal.cpp index 689408201fc..444bad7e5c0 100644 --- a/src/runtime/src/coreclr/pal/src/exception/signal.cpp +++ b/src/runtime/src/coreclr/pal/src/exception/signal.cpp @@ -558,8 +558,13 @@ extern "C" void signal_handler_worker(int code, siginfo_t *siginfo, void *contex // fault. We must disassemble the instruction at record.ExceptionAddress // to correctly fill in this value. - // Unmask the activation signal now that we are running on the original stack of the thread - UnmaskActivationSignal(); + if (code != (SIGSEGV | StackOverflowFlag)) + { + // Unmask the activation signal now that we are running on the original stack of the thread + // except for the stack overflow case when we are actually running on a special stack overflow + // stack. + UnmaskActivationSignal(); + } returnPoint->returnFromHandler = common_signal_handler(code, siginfo, context, 2, (size_t)0, (size_t)siginfo->si_addr); diff --git a/src/runtime/src/coreclr/pal/tests/palsuite/compilableTests.txt b/src/runtime/src/coreclr/pal/tests/palsuite/compilableTests.txt index 9020a18ba2e..be031d72149 100644 --- a/src/runtime/src/coreclr/pal/tests/palsuite/compilableTests.txt +++ b/src/runtime/src/coreclr/pal/tests/palsuite/compilableTests.txt @@ -1,15 +1,5 @@ c_runtime/atof/test1/paltest_atof_test1 c_runtime/atoi/test1/paltest_atoi_test1 -c_runtime/cbrt/test1/paltest_cbrt_test1 -c_runtime/cbrtf/test1/paltest_cbrtf_test1 -c_runtime/ceil/test1/paltest_ceil_test1 -c_runtime/ceilf/test1/paltest_ceilf_test1 -c_runtime/cos/test1/paltest_cos_test1 -c_runtime/cosf/test1/paltest_cosf_test1 -c_runtime/cosh/test1/paltest_cosh_test1 -c_runtime/coshf/test1/paltest_coshf_test1 -c_runtime/errno/test1/paltest_errno_test1 -c_runtime/errno/test2/paltest_errno_test2 c_runtime/isalnum/test1/paltest_isalnum_test1 c_runtime/isalpha/test1/paltest_isalpha_test1 c_runtime/isdigit/test1/paltest_isdigit_test1 @@ -114,14 +104,6 @@ debug_api/OutputDebugStringW/test1/paltest_outputdebugstringw_test1 exception_handling/RaiseException/test1/paltest_raiseexception_test1 exception_handling/RaiseException/test2/paltest_raiseexception_test2 exception_handling/RaiseException/test3/paltest_raiseexception_test3 -filemapping_memmgt/CreateFileMappingA/test1/paltest_createfilemappinga_test1 -filemapping_memmgt/CreateFileMappingA/test3/paltest_createfilemappinga_test3 -filemapping_memmgt/CreateFileMappingA/test4/paltest_createfilemappinga_test4 -filemapping_memmgt/CreateFileMappingA/test5/paltest_createfilemappinga_test5 -filemapping_memmgt/CreateFileMappingA/test6/paltest_createfilemappinga_test6 -filemapping_memmgt/CreateFileMappingA/test7/paltest_createfilemappinga_test7 -filemapping_memmgt/CreateFileMappingA/test8/paltest_createfilemappinga_test8 -filemapping_memmgt/CreateFileMappingA/test9/paltest_createfilemappinga_test9 filemapping_memmgt/CreateFileMappingW/CreateFileMapping_neg1/paltest_createfilemappingw_createfilemapping_neg1 filemapping_memmgt/CreateFileMappingW/test1/paltest_createfilemappingw_test1 filemapping_memmgt/CreateFileMappingW/test3/paltest_createfilemappingw_test3 @@ -187,7 +169,6 @@ file_io/errorpathnotfound/test1/paltest_errorpathnotfound_test1 file_io/errorpathnotfound/test2/paltest_errorpathnotfound_test2 file_io/FILECanonicalizePath/paltest_filecanonicalizepath_test1 file_io/FlushFileBuffers/test1/paltest_flushfilebuffers_test1 -file_io/GetConsoleOutputCP/test1/paltest_getconsoleoutputcp_test1 file_io/GetFileAttributesA/test1/paltest_getfileattributesa_test1 file_io/GetFileAttributesExW/test1/paltest_getfileattributesexw_test1 file_io/GetFileAttributesExW/test2/paltest_getfileattributesexw_test2 @@ -307,8 +288,6 @@ threading/CreateEventW/test2/paltest_createeventw_test2 threading/CreateEventW/test3/paltest_createeventw_test3 threading/CreateMutexW_ReleaseMutex/test1/paltest_createmutexw_releasemutex_test1 threading/CreateMutexW_ReleaseMutex/test2/paltest_createmutexw_releasemutex_test2 -threading/CreateProcessA/test1/paltest_createprocessa_test1 -threading/CreateProcessA/test2/paltest_createprocessa_test2 threading/CreateProcessW/test1/paltest_createprocessw_test1 threading/CreateProcessW/test2/paltest_createprocessw_test2 threading/CreateSemaphoreW_ReleaseSemaphore/test1/paltest_createsemaphorew_releasesemaphore_test1 diff --git a/src/runtime/src/coreclr/pal/tests/palsuite/paltestlist.txt b/src/runtime/src/coreclr/pal/tests/palsuite/paltestlist.txt index a950eaf5787..318009912c9 100644 --- a/src/runtime/src/coreclr/pal/tests/palsuite/paltestlist.txt +++ b/src/runtime/src/coreclr/pal/tests/palsuite/paltestlist.txt @@ -155,7 +155,6 @@ filemapping_memmgt/VirtualQuery/test1/paltest_virtualquery_test1 file_io/errorpathnotfound/test2/paltest_errorpathnotfound_test2 file_io/FILECanonicalizePath/paltest_filecanonicalizepath_test1 file_io/FlushFileBuffers/test1/paltest_flushfilebuffers_test1 -file_io/GetConsoleOutputCP/test1/paltest_getconsoleoutputcp_test1 file_io/GetFileAttributesA/test1/paltest_getfileattributesa_test1 file_io/GetFileAttributesExW/test2/paltest_getfileattributesexw_test2 file_io/GetFileAttributesW/test1/paltest_getfileattributesw_test1 @@ -313,4 +312,3 @@ threading/WaitForSingleObject/WFSOMutexTest/paltest_waitforsingleobject_wfsomute threading/WaitForSingleObject/WFSOSemaphoreTest/paltest_waitforsingleobject_wfsosemaphoretest threading/WaitForSingleObject/WFSOThreadTest/paltest_waitforsingleobject_wfsothreadtest threading/YieldProcessor/test1/paltest_yieldprocessor_test1 -eventprovider/eventprovidertest diff --git a/src/runtime/src/coreclr/pal/tests/palsuite/paltestlist_to_be_reviewed.txt b/src/runtime/src/coreclr/pal/tests/palsuite/paltestlist_to_be_reviewed.txt index 0ee8cb4df9a..f42b770082e 100644 --- a/src/runtime/src/coreclr/pal/tests/palsuite/paltestlist_to_be_reviewed.txt +++ b/src/runtime/src/coreclr/pal/tests/palsuite/paltestlist_to_be_reviewed.txt @@ -1,16 +1,7 @@ This is a list of failing PAL tests that need to be reviewed because. They should either be fixed or deleted if they are no longer applicable. -c_runtime/ferror/test1/paltest_ferror_test1 -c_runtime/ferror/test2/paltest_ferror_test2 -c_runtime/fputs/test2/paltest_fputs_test2 -c_runtime/fread/test1/paltest_fread_test1 -c_runtime/fread/test2/paltest_fread_test2 -c_runtime/fread/test3/paltest_fread_test3 -c_runtime/ftell/test1/paltest_ftell_test1 c_runtime/iswprint/test1/paltest_iswprint_test1 -c_runtime/vprintf/test1/paltest_vprintf_test1 -c_runtime/_getw/test1/paltest_getw_test1 debug_api/DebugBreak/test1/paltest_debugbreak_test1 debug_api/OutputDebugStringA/test1/paltest_outputdebugstringa_test1 debug_api/WriteProcessMemory/test1/paltest_writeprocessmemory_test1 @@ -30,8 +21,6 @@ exception_handling/PAL_EXCEPT_FILTER_EX/test1/paltest_pal_except_filter_ex_test1 exception_handling/PAL_EXCEPT_FILTER_EX/test2/paltest_pal_except_filter_ex_test2 exception_handling/PAL_EXCEPT_FILTER_EX/test3/paltest_pal_except_filter_ex_test3 exception_handling/pal_finally/test1/paltest_pal_finally_test1 -exception_handling/PAL_GetBottommostRegistration/test1/paltest_pal_getbottommostregistration_test1 -exception_handling/PAL_GetBottommostRegistration/test2/paltest_pal_getbottommostregistration_test2 exception_handling/PAL_TRY_EXCEPT/test1/paltest_pal_try_except_test1 exception_handling/PAL_TRY_EXCEPT/test2/paltest_pal_try_except_test2 exception_handling/PAL_TRY_EXCEPT_EX/test1/paltest_pal_try_except_ex_test1 @@ -48,9 +37,6 @@ filemapping_memmgt/GetModuleFileNameA/test1/paltest_getmodulefilenamea_test1 filemapping_memmgt/GetModuleFileNameW/test1/paltest_getmodulefilenamew_test1 filemapping_memmgt/GetProcAddress/test1/paltest_getprocaddress_test1 filemapping_memmgt/GetProcAddress/test2/paltest_getprocaddress_test2 -filemapping_memmgt/ReadProcessMemory/ReadProcessMemory_neg1/paltest_readprocessmemory_readprocessmemory_neg1 -filemapping_memmgt/ReadProcessMemory/test1/paltest_readprocessmemory_test1 -filemapping_memmgt/ReadProcessMemory/test2/paltest_readprocessmemory_test2 file_io/CreateFileA/test1/paltest_createfilea_test1 file_io/CreateFileW/test1/paltest_createfilew_test1 file_io/errorpathnotfound/test1/paltest_errorpathnotfound_test1 @@ -71,8 +57,6 @@ locale_info/CompareStringA/test1/paltest_comparestringa_test1 locale_info/CompareStringW/test1/paltest_comparestringw_test1 locale_info/GetLocaleInfoW/test1/paltest_getlocaleinfow_test1 locale_info/GetLocaleInfoW/test2/paltest_getlocaleinfow_test2 -locale_info/GetStringTypeExW/test1/paltest_getstringtypeexw_test1 -locale_info/GetStringTypeExW/test2/paltest_getstringtypeexw_test2 locale_info/WideCharToMultiByte/test4/paltest_widechartomultibyte_test4 miscellaneous/FormatMessageW/test4/paltest_formatmessagew_test4 miscellaneous/FormatMessageW/test5/paltest_formatmessagew_test5 @@ -83,10 +67,6 @@ miscellaneous/IsBadReadPtr/test1/paltest_isbadreadptr_test1 miscellaneous/IsBadWritePtr/test1/paltest_isbadwriteptr_test1 miscellaneous/IsBadWritePtr/test2/paltest_isbadwriteptr_test2 miscellaneous/IsBadWritePtr/test3/paltest_isbadwriteptr_test3 -miscellaneous/wsprintfW/test2/paltest_wsprintfw_test2 -miscellaneous/wsprintfW/test7/paltest_wsprintfw_test7 -pal_specific/PAL_get_stderr/test1/paltest_pal_get_stderr_test1 -pal_specific/PAL_get_stdin/test1/paltest_pal_get_stdin_test1 pal_specific/PAL_get_stdout/test1/paltest_pal_get_stdout_test1 pal_specific/PAL_RegisterLibraryW_UnregisterLibraryW/test1/paltest_pal_registerlibraryw_unregisterlibraryw_test1 samples/test2/paltest_samples_test2 @@ -112,12 +92,10 @@ threading/OpenEventW/test4/paltest_openeventw_test4 threading/OpenEventW/test5/paltest_openeventw_test5 threading/OpenProcess/test1/paltest_openprocess_test1 threading/QueueUserAPC/test1/paltest_queueuserapc_test1 -threading/setthreadcontext/test1/paltest_setthreadcontext_test1 threading/Sleep/test1/paltest_sleep_test1 threading/SleepEx/test1/paltest_sleepex_test1 threading/SleepEx/test2/paltest_sleepex_test2 threading/TerminateProcess/test1/paltest_terminateprocess_test1 -threading/TLS/test6_optimizedtls/paltest_tls_test6_optimizedtls threading/WaitForMultipleObjectsEx/test5/paltest_waitformultipleobjectsex_test5 threading/WaitForMultipleObjectsEx/test6/paltest_waitformultipleobjectsex_test6 threading/WaitForSingleObject/WFSOProcessTest/paltest_waitforsingleobject_wfsoprocesstest diff --git a/src/runtime/src/coreclr/pal/tests/palsuite/paltests.cpp b/src/runtime/src/coreclr/pal/tests/palsuite/paltests.cpp index 4ec0a3982e3..07e8ff6ece7 100644 --- a/src/runtime/src/coreclr/pal/tests/palsuite/paltests.cpp +++ b/src/runtime/src/coreclr/pal/tests/palsuite/paltests.cpp @@ -54,7 +54,7 @@ int __cdecl main(int argc, char *argv[]) { return PrintTests(argc, argv); } - + PALTest *testCur = PALTest::s_tests; for (;testCur != 0; testCur = testCur->_next) { diff --git a/src/runtime/src/coreclr/scripts/genRuntimeEventSources.py b/src/runtime/src/coreclr/scripts/genRuntimeEventSources.py deleted file mode 100644 index ed13eeec723..00000000000 --- a/src/runtime/src/coreclr/scripts/genRuntimeEventSources.py +++ /dev/null @@ -1,484 +0,0 @@ -# -## Licensed to the .NET Foundation under one or more agreements. -## The .NET Foundation licenses this file to you under the MIT license. -# - -import os -import xml.dom.minidom as DOM -from utilities import open_for_update, parseInclusionList -import argparse -import sys - -generatedCodeFileHeader="""// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -/********************************************************************** - -DO NOT MODIFY. AUTOGENERATED FILE. -This file is generated by /src/coreclr/scripts/genRuntimeEventSources.py - -**********************************************************************/ -""" - -######################################################################## -# START CONFIGURATION -######################################################################## -manifestsToGenerate = { - "Microsoft-Windows-DotNETRuntime" : "NativeRuntimeEventSource.Generated.cs" -} - -providerNameToClassNameMap = { - "Microsoft-Windows-DotNETRuntime" : "NativeRuntimeEventSource" -} - -manifestTypeToCSharpTypeMap = { - "win:UInt8" : "byte", - "win:UInt16" : "ushort", - "win:UInt32" : "uint", - "win:UInt64" : "ulong", - "win:Int32" : "int", - "win:Int64" : "long", - "win:Pointer" : "IntPtr", - "win:UnicodeString" : "string", - "win:Binary" : "byte[]", - "win:Double" : "double", - "win:Boolean" : "bool", - "win:GUID" : "Guid", -} - -overrideEnumBackingTypes = { - "Microsoft-Windows-DotNETRuntime" : { - "GCSuspendEEReasonMap" : "win:UInt32", - "GCRootKindMap" : "win:UInt32" - } -} -######################################################################## -# END CONFIGURATION -######################################################################## - -tabText = "" - -def increaseTabLevel(): - global tabText - tabText += " " - -def decreaseTabLevel(): - global tabText - tabText = tabText[:-4] - -def writeOutput(outputFile, str): - outputFile.write(tabText + str) - -def getCSharpTypeFromManifestType(manifestType): - return manifestTypeToCSharpTypeMap[manifestType] - -def getManifestsToGenerate(): - return manifestsToGenerate - -def includeEvent(inclusionList, providerName, eventName): - if len(inclusionList) == 0: - return True - if providerName in inclusionList and eventName in inclusionList[providerName]: - return True - elif providerName in inclusionList and "*" in inclusionList[providerName]: - return True - elif "*" in inclusionList and eventName in inclusionList["*"]: - return True - elif "*" in inclusionList and "*" in inclusionList["*"]: - return True - else: - return False - -def generateEvent(eventNode, providerNode, outputFile, stringTable): - - # Some threading events are defined manually in NativeRuntimeEventSource.Threading.cs - symbol = eventNode.getAttribute("symbol") - if any(s in symbol for s in ["ThreadPool", "Contention", "WaitHandle"]): - return - - evtLevel = eventNode.getAttribute("level")[4:] - evtKeywords = "" - # Write the event attribute. - writeOutput(outputFile, "[Event("+ eventNode.getAttribute("value") + ", Version = " + eventNode.getAttribute("version") + ", Level = EventLevel." + evtLevel) - - # Not all events have keywords specified, and some have multiple keywords specified. - keywords = eventNode.getAttribute("keywords") - if keywords: - if " " not in keywords: - outputFile.write(", Keywords = Keywords." + keywords) - evtKeywords = "Keywords." + keywords - else: - keywords = keywords.split() - outputFile.write(", Keywords = ") - for keywordIndex in range(len(keywords)): - evtKeywords += "Keywords." + keywords[keywordIndex] - if keywordIndex < (len(keywords) - 1): - evtKeywords += " | " - outputFile.write(evtKeywords) - outputFile.write(")]\n") - - # Get the template for the event. - templateNode = None - templateKey = eventNode.getAttribute("template") - if templateKey is not None: - for node in providerNode.getElementsByTagName("templates"): - templatesNode = node - break - for node in templatesNode.getElementsByTagName("template"): - if node.getAttribute("tid") == templateKey: - templateNode = node - break - - # Write the beginning of the method signature. - writeOutput(outputFile, "private void " + eventNode.getAttribute("symbol") + "(") - - # Write the function signature. - argumentCount = 0 - if templateNode is not None: - argumentNodes = templateNode.childNodes - - # Calculate the number of arguments. - for argumentNode in argumentNodes: - if argumentNode.nodeName == "data": - if argumentNode.getAttribute("inType") != "win:Binary" and argumentNode.getAttribute("inType") != "win:AnsiString" and argumentNode.getAttribute("count") == "": - argumentCount += 1 - else: - break - elif argumentNode.nodeName == "struct": - break - - argumentsProcessed = 0 - for argumentIndex in range(len(argumentNodes)): - argumentNode = argumentNodes[argumentIndex] - if argumentNode.nodeName == "data": - argumentName = argumentNode.getAttribute("name") - argumentInType = argumentNode.getAttribute("inType") - - #### Disable enums until they are needed #### - # argumentMap = argumentNode.getAttribute("map") - # if not argumentMap: - # argumentCSharpType = getCSharpTypeFromManifestType(argumentInType) - # else: - # argumentCSharpType = argumentMap[:-3] - #### Disable enums until they are needed #### - - argumentCSharpType = getCSharpTypeFromManifestType(argumentInType) - outputFile.write(argumentCSharpType + " " + argumentName) - argumentsProcessed += 1 - if argumentsProcessed < argumentCount: - outputFile.write(", ") - if argumentsProcessed == argumentCount: - break - - outputFile.write(")\n") - writeOutput(outputFile, "{\n") - - increaseTabLevel() - writeOutput(outputFile, "// To have this event be emitted from managed side, refer to NativeRuntimeEventSource.cs\n") - writeOutput(outputFile, "throw new NotImplementedException();\n") - decreaseTabLevel() - writeOutput(outputFile, "}\n\n") - - -def generateEvents(providerNode, outputFile, stringTable, inclusion_list): - - providerName = providerNode.getAttribute("name") - - # Get the events element. - for node in providerNode.getElementsByTagName("events"): - eventsNode = node - break - - # Get the list of event nodes. - eventNodes = eventsNode.getElementsByTagName("event") - - # Build a list of events to be emitted. This is where old versions of events are stripped. - # key = eventID, value = version - eventList = dict() - for eventNode in eventNodes: - eventName = eventNode.getAttribute('symbol') - if not includeEvent(inclusion_list, providerName, eventName): - continue - - eventID = eventNode.getAttribute("value") - eventVersion = eventNode.getAttribute("version") - eventList[eventID] = eventVersion - - # Iterate over each event node and process it. - # Only emit events for the latest version of the event, otherwise EventSource initialization will fail. - for eventNode in eventNodes: - eventName = eventNode.getAttribute('symbol') - if not includeEvent(inclusion_list, providerName, eventName): - continue - - eventID = eventNode.getAttribute("value") - eventVersion = eventNode.getAttribute("version") - if eventID in eventList and eventList[eventID] == eventVersion: - generateEvent(eventNode, providerNode, outputFile, stringTable) - elif eventID not in eventList: - raise ValueError("eventID could not be found in the list of events to emit.", eventID) - -def generateValueMapEnums(providerNode, outputFile, stringTable, enumTypeMap): - - # Get the maps element. - for node in providerNode.getElementsByTagName("maps"): - mapsNode = node - break - - # Iterate over each map and create an enum out of it. - for valueMapNode in mapsNode.getElementsByTagName("valueMap"): - - # Get the backing type of the enum. - typeName = enumTypeMap[valueMapNode.getAttribute("name")] - if typeName is None: - raise ValueError("No mapping from mapName to enum backing type.", valueMapNode.getAttribute("name")) - - enumType = getCSharpTypeFromManifestType(typeName) - writeOutput(outputFile, "public enum " + valueMapNode.getAttribute("name")[:-3] + " : " + enumType + "\n") - writeOutput(outputFile, "{\n") - increaseTabLevel() - for mapNode in valueMapNode.getElementsByTagName("map"): - # Each map value has a message, which we should use as the enum value. - messageKey = mapNode.getAttribute("message")[9:-1] - writeOutput(outputFile, stringTable[messageKey] + " = " + mapNode.getAttribute("value") + ",\n") - decreaseTabLevel() - writeOutput(outputFile, "}\n\n") - -def generateBitMapEnums(providerNode, outputFile, stringTable, enumTypeMap): - - # Get the maps element. - for node in providerNode.getElementsByTagName("maps"): - mapsNode = node - break - - # Iterate over each map and create an enum out of it. - for valueMapNode in mapsNode.getElementsByTagName("bitMap"): - - # Get the backing type of the enum. - typeName = enumTypeMap[valueMapNode.getAttribute("name")] - if typeName is None: - raise ValueError("No mapping from mapName to enum backing type.", valueMapNode.getAttribute("name")) - - enumType = getCSharpTypeFromManifestType(typeName) - writeOutput(outputFile, "[Flags]\n") - writeOutput(outputFile, "public enum " + valueMapNode.getAttribute("name")[:-3] + " : " + enumType + "\n") - writeOutput(outputFile, "{\n") - increaseTabLevel() - for mapNode in valueMapNode.getElementsByTagName("map"): - # Each map value has a message, which we should use as the enum value. - messageKey = mapNode.getAttribute("message")[9:-1] - writeOutput(outputFile, stringTable[messageKey] + " = " + mapNode.getAttribute("value") + ",\n") - decreaseTabLevel() - writeOutput(outputFile, "}\n\n") - -def generateEnumTypeMap(providerNode): - - providerName = providerNode.getAttribute("name") - templatesNodes = providerNode.getElementsByTagName("templates") - templatesNode = templatesNodes[0] - mapsNodes = providerNode.getElementsByTagName("maps") - - # Keep a list of mapName -> inType. - # This map contains the first inType seen for the specified mapName. - typeMap = dict() - - # There are a couple of maps that are used by multiple events but have different backing types. - # Because only one of the uses will be consumed by EventSource/EventListener we can hack the backing type here - # and suppress the warning that we'd otherwise get. - overrideTypeMap = dict() - if providerName in overrideEnumBackingTypes: - overrideTypeMap = overrideEnumBackingTypes[providerName] - - for mapsNode in mapsNodes: - for valueMapNode in mapsNode.getElementsByTagName("valueMap"): - mapName = valueMapNode.getAttribute("name") - dataNodes = templatesNode.getElementsByTagName("data") - - # If we've never seen the map used, save its usage with the inType. - # If we have seen the map used, make sure that the inType saved previously matches the current inType. - for dataNode in dataNodes: - if dataNode.getAttribute("map") == mapName: - if mapName in overrideTypeMap: - typeMap[mapName] = overrideTypeMap[mapName] - elif mapName in typeMap and typeMap[mapName] != dataNode.getAttribute("inType"): - print("WARNING: Map " + mapName + " is used multiple times with different types. This may cause functional bugs in tracing.") - elif not mapName in typeMap: - typeMap[mapName] = dataNode.getAttribute("inType") - for bitMapNode in mapsNode.getElementsByTagName("bitMap"): - mapName = bitMapNode.getAttribute("name") - dataNodes = templatesNode.getElementsByTagName("data") - - # If we've never seen the map used, save its usage with the inType. - # If we have seen the map used, make sure that the inType saved previously matches the current inType. - for dataNode in dataNodes: - if dataNode.getAttribute("map") == mapName: - if mapName in overrideTypeMap: - typeMap[mapName] = overrideTypeMap[mapName] - elif mapName in typeMap and typeMap[mapName] != dataNode.getAttribute("inType"): - print("Map " + mapName + " is used multiple times with different types.") - elif not mapName in typeMap: - typeMap[mapName] = dataNode.getAttribute("inType") - - return typeMap - -def generateKeywordsClass(providerNode, outputFile, inclusion_list): - - providerName = providerNode.getAttribute("name") - - # Get the events element. - for node in providerNode.getElementsByTagName("events"): - eventsNode = node - break - - # Get the list of event nodes. - eventNodes = eventsNode.getElementsByTagName("event") - - # Build the list of used keywords - keywordSet = set() - for eventNode in eventNodes: - eventName = eventNode.getAttribute('symbol') - if not includeEvent(inclusion_list, providerName, eventName): - continue - - # Not all events have keywords specified, and some have multiple keywords specified. - keywords = eventNode.getAttribute("keywords") - if keywords: - keywordSet = keywordSet.union(keywords.split()) - - # Find the keywords element. - for node in providerNode.getElementsByTagName("keywords"): - keywordsNode = node - break; - - writeOutput(outputFile, "public static class Keywords\n") - writeOutput(outputFile, "{\n") - increaseTabLevel() - - for keywordNode in keywordsNode.getElementsByTagName("keyword"): - keywordName = keywordNode.getAttribute("name") - if keywordName not in keywordSet: - continue; - - writeOutput(outputFile, "public const EventKeywords " + keywordName + " = (EventKeywords)" + keywordNode.getAttribute("mask") + ";\n") - - decreaseTabLevel() - writeOutput(outputFile, "}\n\n") - -def loadStringTable(manifest): - - # Create the string table dictionary. - stringTable = dict() - - # Get the string table element. - for node in manifest.getElementsByTagName("stringTable"): - stringTableNode = node - break - - # Iterate through each string and save it. - for stringElem in stringTableNode.getElementsByTagName("string"): - stringTable[stringElem.getAttribute("id")] = stringElem.getAttribute("value") - - return stringTable - -def generateEventSources(manifestFullPath, intermediatesDirFullPath, inclusion_list): - - # Open the manifest for reading. - manifest = DOM.parse(manifestFullPath) - - # Load the string table. - stringTable = loadStringTable(manifest) - - # Iterate over each provider that we want to generate an EventSource for. - for providerName, outputFileName in getManifestsToGenerate().items(): - for node in manifest.getElementsByTagName("provider"): - if node.getAttribute("name") == providerName: - providerNode = node - break - - if providerNode is None: - raise ValueError("Unable to find provider node.", providerName) - - # Generate a full path to the output file and open the file for open_for_update. - outputFilePath = os.path.join(intermediatesDirFullPath, outputFileName) - with open_for_update(outputFilePath) as outputFile: - - # Write the license header. - writeOutput(outputFile, generatedCodeFileHeader) - - # Write the class header. - header = """ -using System; - -namespace System.Diagnostics.Tracing -{ -""" - writeOutput(outputFile, header) - increaseTabLevel() - - className = providerNameToClassNameMap[providerName] - writeOutput(outputFile, "internal sealed partial class " + className + " : EventSource\n") - writeOutput(outputFile, "{\n") - increaseTabLevel() - - # Write the keywords class. - generateKeywordsClass(providerNode, outputFile, inclusion_list) - - #### Disable enums until they are needed #### - # Generate the enum type map. - # This determines what the backing type for each enum should be. - # enumTypeMap = generateEnumTypeMap(providerNode) - - # Generate enums for value maps. - # generateValueMapEnums(providerNode, outputFile, stringTable, enumTypeMap) - - # Generate enums for bit maps. - # generateBitMapEnums(providerNode, outputFile, stringTable, enumTypeMap) - #### Disable enums until they are needed #### - - # Generate events. - generateEvents(providerNode, outputFile, stringTable, inclusion_list) - - # Write the class footer. - decreaseTabLevel() - writeOutput(outputFile, "}\n") - decreaseTabLevel() - writeOutput(outputFile, "}\n") - -def main(argv): - - # Parse command line arguments. - parser = argparse.ArgumentParser( - description="Generates C# EventSource classes that represent the runtime's native event providers.") - - required = parser.add_argument_group('required arguments') - required.add_argument('--man', type=str, required=True, - help='full path to manifest containing the description of events') - required.add_argument('--intermediate', type=str, required=True, - help='full path to eventprovider intermediate directory') - required.add_argument('--inc', type=str,default="", - help='full path to inclusion list') - args, unknown = parser.parse_known_args(argv) - if unknown: - print('Unknown argument(s): ', ', '.join(unknown)) - return 1 - - manifestFullPath = args.man - intermediatesDirFullPath = args.intermediate - inclusion_filename = args.inc - - # Ensure the intermediates directory exists. - try: - os.makedirs(intermediatesDirFullPath) - except OSError: - if not os.path.isdir(intermediatesDirFullPath): - raise - - inclusion_list = parseInclusionList(inclusion_filename) - - # Generate event sources. - generateEventSources(manifestFullPath, intermediatesDirFullPath, inclusion_list) - return 0 - -if __name__ == '__main__': - return_code = main(sys.argv[1:]) - sys.exit(return_code) diff --git a/src/runtime/src/coreclr/tools/Common/Internal/Runtime/RiscVLoongArch64FpStruct.cs b/src/runtime/src/coreclr/tools/Common/Internal/Runtime/RiscVLoongArch64FpStruct.cs index ad066f1d9ec..577f60deb6c 100644 --- a/src/runtime/src/coreclr/tools/Common/Internal/Runtime/RiscVLoongArch64FpStruct.cs +++ b/src/runtime/src/coreclr/tools/Common/Internal/Runtime/RiscVLoongArch64FpStruct.cs @@ -78,7 +78,8 @@ private static void SetFpStructInRegistersInfoField(ref FpStructInRegistersInfo (index == 0 ? ref info.offset1st : ref info.offset2nd) = offset; } - private static bool HandleInlineArray(int elementTypeIndex, int nElements, ref FpStructInRegistersInfo info, ref int typeIndex) + private static bool HandleInlineArray(int elementTypeIndex, int nElements, + ref FpStructInRegistersInfo info, ref int typeIndex, ref uint occupiedBytesMap) { int nFlattenedFieldsPerElement = typeIndex - elementTypeIndex; if (nFlattenedFieldsPerElement == 0) @@ -110,6 +111,16 @@ private static bool HandleInlineArray(int elementTypeIndex, int nElements, ref F int sizeShiftMask = (int)(info.flags & FpStruct.SizeShift1stMask) << 2; info.flags |= (FpStruct)(floatFlag | sizeShiftMask); // merge with 1st field info.offset2nd = info.offset1st + info.Size1st(); // bump up the field offset + + Debug.Assert(info.Size1st() == info.Size2nd()); + uint startOffset = info.offset2nd; + uint endOffset = startOffset + info.Size2nd(); + + uint fieldOccupation = (~0u << (int)startOffset) ^ (~0u << (int)endOffset); + if ((occupiedBytesMap & fieldOccupation) != 0) + return false; // duplicated array element overlaps with other fields + + occupiedBytesMap |= fieldOccupation; } return true; } @@ -119,23 +130,30 @@ private static bool FlattenFields(TypeDesc td, uint offset, ref FpStructInRegist IEnumerable fields = td.GetFields(); int nFields = 0; int elementTypeIndex = typeIndex; - FieldDesc prevField = null; + FieldDesc lastField = null; + uint occupiedBytesMap = 0; foreach (FieldDesc field in fields) { if (field.IsStatic) continue; nFields++; - if (prevField != null && prevField.Offset.AsInt + prevField.FieldType.GetElementSize().AsInt > field.Offset.AsInt) + uint startOffset = offset + (uint)field.Offset.AsInt; + uint endOffset = startOffset + (uint)field.FieldType.GetElementSize().AsInt; + + uint fieldOccupation = (~0u << (int)startOffset) ^ (~0u << (int)endOffset); + if ((occupiedBytesMap & fieldOccupation) != 0) return false; // fields overlap, treat as union - prevField = field; + occupiedBytesMap |= fieldOccupation; + + lastField = field; TypeFlags category = field.FieldType.Category; if (category == TypeFlags.ValueType) { TypeDesc nested = field.FieldType; - if (!FlattenFields(nested, offset + (uint)field.Offset.AsInt, ref info, ref typeIndex)) + if (!FlattenFields(nested, startOffset, ref info, ref typeIndex)) return false; } else if (field.FieldType.GetElementSize().AsInt <= TARGET_POINTER_SIZE) @@ -145,7 +163,7 @@ private static bool FlattenFields(TypeDesc td, uint offset, ref FpStructInRegist bool isFloating = category is TypeFlags.Single or TypeFlags.Double; SetFpStructInRegistersInfoField(ref info, typeIndex++, - isFloating, (uint)field.FieldType.GetElementSize().AsInt, offset + (uint)field.Offset.AsInt); + isFloating, (uint)field.FieldType.GetElementSize().AsInt, startOffset); } else { @@ -156,7 +174,7 @@ private static bool FlattenFields(TypeDesc td, uint offset, ref FpStructInRegist if ((td as MetadataType).HasImpliedRepeatedFields()) { Debug.Assert(nFields == 1); - int nElements = td.GetElementSize().AsInt / prevField.FieldType.GetElementSize().AsInt; + int nElements = td.GetElementSize().AsInt / lastField.FieldType.GetElementSize().AsInt; // Only InlineArrays can have element type of empty struct, fixed-size buffers take only primitives if ((typeIndex - elementTypeIndex) == 0 && (td as MetadataType).IsInlineArray) @@ -165,7 +183,7 @@ private static bool FlattenFields(TypeDesc td, uint offset, ref FpStructInRegist return false; // struct containing an array of empty structs is passed by integer calling convention } - if (!HandleInlineArray(elementTypeIndex, nElements, ref info, ref typeIndex)) + if (!HandleInlineArray(elementTypeIndex, nElements, ref info, ref typeIndex, ref occupiedBytesMap)) return false; } return true; @@ -189,6 +207,20 @@ public static FpStructInRegistersInfo GetFpStructInRegistersInfo(TypeDesc td, Ta return new FpStructInRegistersInfo{}; // struct has no floating fields Debug.Assert(nFields == 1 || nFields == 2); + if (nFields == 2 && info.offset1st > info.offset2nd) + { + // swap fields to match memory order + info.flags = (FpStruct)( + ((uint)(info.flags & FloatInt) << (PosIntFloat - PosFloatInt)) | + ((uint)(info.flags & IntFloat) >> (PosIntFloat - PosFloatInt)) | + ((uint)(info.flags & SizeShift1stMask) << (PosSizeShift2nd - PosSizeShift1st)) | + ((uint)(info.flags & SizeShift2ndMask) >> (PosSizeShift2nd - PosSizeShift1st)) + ); + (info.offset2nd, info.offset1st) = (info.offset1st, info.offset2nd); + } + Debug.Assert((info.flags & (OnlyOne | BothFloat)) == 0); + Debug.Assert((info.flags & FloatInt) == 0 || info.Size1st() == sizeof(float) || info.Size1st() == sizeof(double)); + Debug.Assert((info.flags & IntFloat) == 0 || info.Size2nd() == sizeof(float) || info.Size2nd() == sizeof(double)); if ((info.flags & (FloatInt | IntFloat)) == (FloatInt | IntFloat)) { diff --git a/src/runtime/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/R2RPEBuilder.cs b/src/runtime/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/R2RPEBuilder.cs index 7a8aa110e7f..738910d98e9 100644 --- a/src/runtime/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/R2RPEBuilder.cs +++ b/src/runtime/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/R2RPEBuilder.cs @@ -28,52 +28,11 @@ public sealed class R2RPEBuilder : PEBuilder /// const int RVABitsToMatchFilePos = 16; - /// - /// This structure describes how a particular section moved between the original MSIL - /// and the output PE file. It holds beginning and end RVA of the input (MSIL) section - /// and a delta between the input and output starting RVA of the section. - /// - struct SectionRVADelta - { - /// - /// Starting RVA of the section in the input MSIL PE. - /// - public readonly int StartRVA; - - /// - /// End RVA (one plus the last RVA in the section) of the section in the input MSIL PE. - /// - public readonly int EndRVA; - - /// - /// Starting RVA of the section in the output PE minus its starting RVA in the input MSIL. - /// - public readonly int DeltaRVA; - - /// - /// Initialize the section RVA delta information. - /// - /// Starting RVA of the section in the input MSIL - /// End RVA of the section in the input MSIL - /// Output RVA of the section minus input RVA of the section - public SectionRVADelta(int startRVA, int endRVA, int deltaRVA) - { - StartRVA = startRVA; - EndRVA = endRVA; - DeltaRVA = deltaRVA; - } - } - /// /// Name of the text section. /// public const string TextSectionName = ".text"; - /// - /// Name of the initialized data section. - /// - public const string SDataSectionName = ".sdata"; - /// /// Name of the relocation section. /// @@ -100,13 +59,6 @@ public SectionRVADelta(int startRVA, int endRVA, int deltaRVA) /// private Func _getRuntimeFunctionsTable; - /// - /// For each copied section, we store its initial and end RVA in the source PE file - /// and the RVA difference between the old and new file. We use this table to relocate - /// directory entries in the PE file header. - /// - private List _sectionRvaDeltas; - private class SerializedSectionData { /// @@ -194,7 +146,6 @@ public R2RPEBuilder( { _target = target; _getRuntimeFunctionsTable = getRuntimeFunctionsTable; - _sectionRvaDeltas = new List(); _sectionBuilder = new SectionBuilder(target); @@ -210,16 +161,13 @@ public R2RPEBuilder( _sectionBuilder.SetDllNameForExportDirectoryTable(outputFileSimpleName); } - if (_sectionBuilder.FindSection(R2RPEBuilder.RelocSectionName) == null) - { - // Always inject the relocation section to the end of section list - _sectionBuilder.AddSection( - R2RPEBuilder.RelocSectionName, - SectionCharacteristics.ContainsInitializedData | - SectionCharacteristics.MemRead | - SectionCharacteristics.MemDiscardable, - PEHeaderConstants.SectionAlignment); - } + // Always inject the relocation section to the end of section list + _sectionBuilder.AddSection( + R2RPEBuilder.RelocSectionName, + SectionCharacteristics.ContainsInitializedData | + SectionCharacteristics.MemRead | + SectionCharacteristics.MemDiscardable, + PEHeaderConstants.SectionAlignment); List sectionData = new List(); foreach (SectionInfo sectionInfo in _sectionBuilder.GetSections()) @@ -353,7 +301,7 @@ public void AddSections(OutputInfoBuilder outputInfoBuilder) sizeof(int) + // SizeOfUninitializedData sizeof(int) + // AddressOfEntryPoint sizeof(int) + // BaseOfCode - sizeof(long); // PE32: BaseOfData (int), ImageBase (int) + sizeof(long); // PE32: BaseOfData (int), ImageBase (int) // PE32+: ImageBase (long) const int OffsetOfChecksum = OffsetOfSectionAlign + sizeof(int) + // SectionAlignment @@ -371,7 +319,7 @@ public void AddSections(OutputInfoBuilder outputInfoBuilder) const int OffsetOfSizeOfImage = OffsetOfChecksum - 2 * sizeof(int); // SizeOfHeaders, SizeOfImage const int SectionHeaderNameSize = 8; - const int SectionHeaderVirtualSize = SectionHeaderNameSize; // VirtualSize follows + const int SectionHeaderVirtualSize = SectionHeaderNameSize; // VirtualSize follows const int SectionHeaderRVAOffset = SectionHeaderVirtualSize + sizeof(int); // RVA Offset follows VirtualSize + 4 bytes VirtualSize const int SectionHeaderSizeOfRawData = SectionHeaderRVAOffset + sizeof(int); // SizeOfRawData follows RVA const int SectionHeaderPointerToRawDataOffset = SectionHeaderSizeOfRawData + sizeof(int); // PointerToRawData immediately follows the SizeOfRawData @@ -385,7 +333,7 @@ public void AddSections(OutputInfoBuilder outputInfoBuilder) sizeof(int) + // PointerToRelocations sizeof(int) + // PointerToLineNumbers sizeof(short) + // NumberOfRelocations - sizeof(short) + // NumberOfLineNumbers + sizeof(short) + // NumberOfLineNumbers sizeof(int); // SectionCharacteristics /// @@ -538,42 +486,8 @@ protected override PEDirectoriesBuilder GetDirectories() size: runtimeFunctionsTable.TableSizeExcludingSentinel); } } - - return builder; - } - /// - /// Relocate a single directory entry. - /// - /// Directory entry to allocate - /// Relocated directory entry - public DirectoryEntry RelocateDirectoryEntry(DirectoryEntry entry) - { - return new DirectoryEntry(RelocateRVA(entry.RelativeVirtualAddress), entry.Size); - } - - /// - /// Relocate a given RVA using the section offset table produced during section serialization. - /// - /// RVA to relocate - /// Relocated RVA - private int RelocateRVA(int rva) - { - if (rva == 0) - { - // Zero RVA is normally used as NULL - return rva; - } - foreach (SectionRVADelta sectionRvaDelta in _sectionRvaDeltas) - { - if (rva >= sectionRvaDelta.StartRVA && rva < sectionRvaDelta.EndRVA) - { - // We found the input section holding the RVA, apply its specific delt (output RVA - input RVA). - return rva + sectionRvaDelta.DeltaRVA; - } - } - Debug.Fail("RVA is not within any of the input sections - output PE may be inconsistent"); - return rva; + return builder; } /// diff --git a/src/runtime/src/coreclr/vm/methodtable.cpp b/src/runtime/src/coreclr/vm/methodtable.cpp index e04484a7427..170222cf3b9 100644 --- a/src/runtime/src/coreclr/vm/methodtable.cpp +++ b/src/runtime/src/coreclr/vm/methodtable.cpp @@ -2716,8 +2716,8 @@ static void SetFpStructInRegistersInfoField(FpStructInRegistersInfo& info, int i (index == 0 ? info.offset1st : info.offset2nd) = offset; } -static bool HandleInlineArray(int elementTypeIndex, int nElements, FpStructInRegistersInfo& info, int& typeIndex - DEBUG_ARG(int nestingLevel)) +static bool HandleInlineArray(int elementTypeIndex, int nElements, + FpStructInRegistersInfo& info, int& typeIndex, uint32_t& occupiedBytesMap DEBUG_ARG(int nestingLevel)) { int nFlattenedFieldsPerElement = typeIndex - elementTypeIndex; if (nFlattenedFieldsPerElement == 0) @@ -2759,45 +2759,65 @@ static bool HandleInlineArray(int elementTypeIndex, int nElements, FpStructInReg int sizeShiftMask = (info.flags & FpStruct::SizeShift1stMask) << 2; info.flags = FpStruct::Flags(info.flags | floatFlag | sizeShiftMask); // merge with 1st field info.offset2nd = info.offset1st + info.Size1st(); // bump up the field offset + + assert(info.Size1st() == info.Size2nd()); + uint32_t startOffset = info.offset2nd; + uint32_t endOffset = startOffset + info.Size2nd(); + + uint32_t fieldOccupation = (~0u << startOffset) ^ (~0u << endOffset); + if ((occupiedBytesMap & fieldOccupation) != 0) + { + LOG((LF_JIT, LL_EVERYTHING, "FpStructInRegistersInfo:%*s " + " * duplicated array element [%i..%i) overlaps with other fields (occupied bytes map: 0x%04x), treat as union\n", + nestingLevel * 4, "", startOffset, endOffset, occupiedBytesMap)); + return false; + } + occupiedBytesMap |= fieldOccupation; + LOG((LF_JIT, LL_EVERYTHING, "FpStructInRegistersInfo:%*s * duplicated array element type\n", nestingLevel * 4, "")); } return true; } -static bool FlattenFields(TypeHandle th, uint32_t offset, FpStructInRegistersInfo& info, int& typeIndex +static bool FlattenFields(TypeHandle th, uint32_t structOffset, FpStructInRegistersInfo& info, int& typeIndex DEBUG_ARG(int nestingLevel)) { bool isManaged = !th.IsTypeDesc(); MethodTable* pMT = isManaged ? th.AsMethodTable() : th.AsNativeValueType(); int nFields = isManaged ? pMT->GetNumIntroducedInstanceFields() : pMT->GetNativeLayoutInfo()->GetNumFields(); - LOG((LF_JIT, LL_EVERYTHING, "FpStructInRegistersInfo:%*s flattening %s (%s, %i fields)\n", - nestingLevel * 4, "", pMT->GetDebugClassName(), (isManaged ? "managed" : "native"), nFields)); + LOG((LF_JIT, LL_EVERYTHING, "FpStructInRegistersInfo:%*s flattening %s (%s, %i fields) at offset %u\n", + nestingLevel * 4, "", pMT->GetDebugClassName(), (isManaged ? "managed" : "native"), nFields, structOffset)); // TODO: templatize isManaged and use if constexpr for differences when we migrate to C++17 // because the logic for both branches is nearly the same. + + uint32_t occupiedBytesMap = 0; if (isManaged) { FieldDesc* fields = pMT->GetApproxFieldDescListRaw(); int elementTypeIndex = typeIndex; for (int i = 0; i < nFields; ++i) { - if (i > 0 && fields[i-1].GetOffset() + fields[i-1].GetSize() > fields[i].GetOffset()) + uint32_t startOffset = structOffset + fields[i].GetOffset(); + uint32_t endOffset = startOffset + fields[i].GetSize(); + + uint32_t fieldOccupation = (~0u << startOffset) ^ (~0u << endOffset); + if ((occupiedBytesMap & fieldOccupation) != 0) { LOG((LF_JIT, LL_EVERYTHING, "FpStructInRegistersInfo:%*s " - " * fields %s [%i..%i) and %s [%i..%i) overlap, treat as union\n", - nestingLevel * 4, "", - fields[i-1].GetDebugName(), fields[i-1].GetOffset(), fields[i-1].GetOffset() + fields[i-1].GetSize(), - fields[i].GetDebugName(), fields[i].GetOffset(), fields[i].GetOffset() + fields[i].GetSize())); + " * field %s [%i..%i) overlaps with other fields (occupied bytes map: 0x%04x), treat as union\n", + nestingLevel * 4, "", fields[i].GetDebugName(), startOffset, endOffset, occupiedBytesMap)); return false; } + occupiedBytesMap |= fieldOccupation; CorElementType type = fields[i].GetFieldType(); if (type == ELEMENT_TYPE_VALUETYPE) { MethodTable* nested = fields[i].GetApproxFieldTypeHandleThrowing().GetMethodTable(); - if (!FlattenFields(TypeHandle(nested), offset + fields[i].GetOffset(), info, typeIndex DEBUG_ARG(nestingLevel + 1))) + if (!FlattenFields(TypeHandle(nested), startOffset, info, typeIndex DEBUG_ARG(nestingLevel + 1))) return false; } else if (fields[i].GetSize() <= TARGET_POINTER_SIZE) @@ -2810,12 +2830,10 @@ static bool FlattenFields(TypeHandle th, uint32_t offset, FpStructInRegistersInf } bool isFloating = CorTypeInfo::IsFloat_NoThrow(type); - SetFpStructInRegistersInfoField(info, typeIndex++, - isFloating, CorTypeInfo::Size_NoThrow(type), offset + fields[i].GetOffset()); + SetFpStructInRegistersInfoField(info, typeIndex++, isFloating, CorTypeInfo::Size_NoThrow(type), startOffset); LOG((LF_JIT, LL_EVERYTHING, "FpStructInRegistersInfo:%*s * found field %s [%i..%i), type: %s\n", - nestingLevel * 4, "", fields[i].GetDebugName(), - fields[i].GetOffset(), fields[i].GetOffset() + fields[i].GetSize(), CorTypeInfo::GetName(type))); + nestingLevel * 4, "", fields[i].GetDebugName(), startOffset, endOffset, CorTypeInfo::GetName(type))); } else { @@ -2842,7 +2860,7 @@ static bool FlattenFields(TypeHandle th, uint32_t offset, FpStructInRegistersInf return false; } - if (!HandleInlineArray(elementTypeIndex, nElements, info, typeIndex DEBUG_ARG(nestingLevel + 1))) + if (!HandleInlineArray(elementTypeIndex, nElements, info, typeIndex, occupiedBytesMap DEBUG_ARG(nestingLevel + 1))) return false; } } @@ -2851,15 +2869,18 @@ static bool FlattenFields(TypeHandle th, uint32_t offset, FpStructInRegistersInf const NativeFieldDescriptor* fields = pMT->GetNativeLayoutInfo()->GetNativeFieldDescriptors(); for (int i = 0; i < nFields; ++i) { - if (i > 0 && fields[i-1].GetExternalOffset() + fields[i-1].NativeSize() > fields[i].GetExternalOffset()) + uint32_t startOffset = structOffset + fields[i].GetExternalOffset(); + uint32_t endOffset = startOffset + fields[i].NativeSize(); + + uint32_t fieldOccupation = (~0u << startOffset) ^ (~0u << endOffset); + if ((occupiedBytesMap & fieldOccupation) != 0) { LOG((LF_JIT, LL_EVERYTHING, "FpStructInRegistersInfo:%*s " - " * fields %s [%i..%i) and %s [%i..%i) overlap, treat as union\n", - nestingLevel * 4, "", - fields[i-1].GetFieldDesc()->GetDebugName(), fields[i-1].GetExternalOffset(), fields[i-1].GetExternalOffset() + fields[i-1].NativeSize(), - fields[i].GetFieldDesc()->GetDebugName(), fields[i].GetExternalOffset(), fields[i].GetExternalOffset() + fields[i].NativeSize())); + " * field %s [%i..%i) overlaps with other fields (occupied bytes map: 0x%04x), treat as union\n", + nestingLevel * 4, "", fields[i].GetFieldDesc()->GetDebugName(), startOffset, endOffset)); return false; } + occupiedBytesMap |= fieldOccupation; static const char* categoryNames[] = {"FLOAT", "NESTED", "INTEGER", "ILLEGAL"}; NativeFieldCategory category = fields[i].GetCategory(); @@ -2868,12 +2889,12 @@ static bool FlattenFields(TypeHandle th, uint32_t offset, FpStructInRegistersInf int elementTypeIndex = typeIndex; MethodTable* nested = fields[i].GetNestedNativeMethodTable(); - if (!FlattenFields(TypeHandle(nested), offset + fields[i].GetExternalOffset(), info, typeIndex DEBUG_ARG(nestingLevel + 1))) + if (!FlattenFields(TypeHandle(nested), startOffset, info, typeIndex DEBUG_ARG(nestingLevel + 1))) return false; // In native layout fixed arrays are marked as NESTED just like structs int nElements = fields[i].GetNumElements(); - if (!HandleInlineArray(elementTypeIndex, nElements, info, typeIndex DEBUG_ARG(nestingLevel + 1))) + if (!HandleInlineArray(elementTypeIndex, nElements, info, typeIndex, occupiedBytesMap DEBUG_ARG(nestingLevel + 1))) return false; } else if (fields[i].NativeSize() <= TARGET_POINTER_SIZE) @@ -2886,13 +2907,10 @@ static bool FlattenFields(TypeHandle th, uint32_t offset, FpStructInRegistersInf } bool isFloating = (category == NativeFieldCategory::FLOAT); - - SetFpStructInRegistersInfoField(info, typeIndex++, - isFloating, fields[i].NativeSize(), offset + fields[i].GetExternalOffset()); + SetFpStructInRegistersInfoField(info, typeIndex++, isFloating, fields[i].NativeSize(), startOffset); LOG((LF_JIT, LL_EVERYTHING, "FpStructInRegistersInfo:%*s * found field %s [%i..%i), type: %s\n", - nestingLevel * 4, "", fields[i].GetFieldDesc()->GetDebugName(), - fields[i].GetExternalOffset(), fields[i].GetExternalOffset() + fields[i].NativeSize(), categoryNames[(int)category])); + nestingLevel * 4, "", fields[i].GetFieldDesc()->GetDebugName(), startOffset, endOffset, categoryNames[(int)category])); } else { @@ -2930,6 +2948,21 @@ FpStructInRegistersInfo MethodTable::GetFpStructInRegistersInfo(TypeHandle th) } assert(nFields == 1 || nFields == 2); + if (nFields == 2 && info.offset1st > info.offset2nd) + { + LOG((LF_JIT, LL_EVERYTHING, "FpStructInRegistersInfo: struct %s (%u bytes): swap fields to match memory order\n", + (!th.IsTypeDesc() ? th.AsMethodTable() : th.AsNativeValueType())->GetDebugClassName(), th.GetSize())); + info.flags = FpStruct::Flags( + ((info.flags & FloatInt) << (PosIntFloat - PosFloatInt)) | + ((info.flags & IntFloat) >> (PosIntFloat - PosFloatInt)) | + ((info.flags & SizeShift1stMask) << (PosSizeShift2nd - PosSizeShift1st)) | + ((info.flags & SizeShift2ndMask) >> (PosSizeShift2nd - PosSizeShift1st)) + ); + std::swap(info.offset1st, info.offset2nd); + } + assert((info.flags & (OnlyOne | BothFloat)) == 0); + assert((info.flags & FloatInt) == 0 || info.Size1st() == sizeof(float) || info.Size1st() == sizeof(double)); + assert((info.flags & IntFloat) == 0 || info.Size2nd() == sizeof(float) || info.Size2nd() == sizeof(double)); if ((info.flags & (FloatInt | IntFloat)) == (FloatInt | IntFloat)) { diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/ref/Microsoft.Extensions.FileSystemGlobbing.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/ref/Microsoft.Extensions.FileSystemGlobbing.cs index d06828fc9d7..600e3df3aa5 100644 --- a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/ref/Microsoft.Extensions.FileSystemGlobbing.cs +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/ref/Microsoft.Extensions.FileSystemGlobbing.cs @@ -31,6 +31,7 @@ public partial class Matcher { public Matcher() { } public Matcher(System.StringComparison comparisonType) { } + public Matcher(System.StringComparison comparisonType = System.StringComparison.OrdinalIgnoreCase, bool preserveFilterOrder = false) { } public virtual Microsoft.Extensions.FileSystemGlobbing.Matcher AddExclude(string pattern) { throw null; } public virtual Microsoft.Extensions.FileSystemGlobbing.Matcher AddInclude(string pattern) { throw null; } public virtual Microsoft.Extensions.FileSystemGlobbing.PatternMatchingResult Execute(Microsoft.Extensions.FileSystemGlobbing.Abstractions.DirectoryInfoBase directoryInfo) { throw null; } diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/IncludeOrExcludeValue.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/IncludeOrExcludeValue.cs new file mode 100644 index 00000000000..ce295f6d08f --- /dev/null +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/IncludeOrExcludeValue.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.FileSystemGlobbing.Internal +{ + internal struct IncludeOrExcludeValue + { + internal TValue Value; + internal bool IsInclude; + } +} diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/MatcherContext.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/MatcherContext.cs index 5080120689d..4ba7b989274 100644 --- a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/MatcherContext.cs +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/MatcherContext.cs @@ -6,6 +6,7 @@ using System.Linq; using Microsoft.Extensions.FileSystemGlobbing.Abstractions; using Microsoft.Extensions.FileSystemGlobbing.Internal.PathSegments; +using Microsoft.Extensions.FileSystemGlobbing.Internal.PatternContexts; using Microsoft.Extensions.FileSystemGlobbing.Util; namespace Microsoft.Extensions.FileSystemGlobbing.Internal @@ -17,8 +18,7 @@ namespace Microsoft.Extensions.FileSystemGlobbing.Internal public class MatcherContext { private readonly DirectoryInfoBase _root; - private readonly IPatternContext[] _includePatternContexts; - private readonly IPatternContext[] _excludePatternContexts; + private readonly IPatternContext _patternContext; private readonly List _files; private readonly HashSet _declaredLiteralFolderSegmentInString; @@ -27,22 +27,33 @@ public class MatcherContext private bool _declaredParentPathSegment; private bool _declaredWildcardPathSegment; - private readonly StringComparison _comparisonType; - - public MatcherContext( - IEnumerable includePatterns, - IEnumerable excludePatterns, - DirectoryInfoBase directoryInfo, - StringComparison comparison) + public MatcherContext(IEnumerable includePatterns, IEnumerable excludePatterns, DirectoryInfoBase directoryInfo, StringComparison comparison) { _root = directoryInfo; - _files = new List(); - _comparisonType = comparison; + _files = []; + _declaredLiteralFolderSegmentInString = new HashSet(StringComparisonHelper.GetStringComparer(comparison)); - _includePatternContexts = includePatterns.Select(pattern => pattern.CreatePatternContextForInclude()).ToArray(); - _excludePatternContexts = excludePatterns.Select(pattern => pattern.CreatePatternContextForExclude()).ToArray(); + IPatternContext[] includePatternContexts = includePatterns.Select(pattern => pattern.CreatePatternContextForInclude()).ToArray(); + IPatternContext[] excludePatternContexts = excludePatterns.Select(pattern => pattern.CreatePatternContextForExclude()).ToArray(); + + _patternContext = new IncludesFirstCompositePatternContext(includePatternContexts, excludePatternContexts); + } + internal MatcherContext(List> orderedPatterns, DirectoryInfoBase directoryInfo, StringComparison comparison) + { + _root = directoryInfo; + _files = []; _declaredLiteralFolderSegmentInString = new HashSet(StringComparisonHelper.GetStringComparer(comparison)); + + IncludeOrExcludeValue[] includeOrExcludePatternContexts = orderedPatterns + .Select(item => new IncludeOrExcludeValue + { + Value = item.IsInclude ? item.Value.CreatePatternContextForInclude() : item.Value.CreatePatternContextForExclude(), + IsInclude = item.IsInclude + }) + .ToArray(); + + _patternContext = new PreserveOrderCompositePatternContext(includeOrExcludePatternContexts); } public PatternMatchingResult Execute() @@ -57,7 +68,7 @@ public PatternMatchingResult Execute() private void Match(DirectoryInfoBase directory, string? parentRelativePath) { // Request all the including and excluding patterns to push current directory onto their status stack. - PushDirectory(directory); + _patternContext.PushDirectory(directory); Declare(); var entities = new List(); @@ -89,7 +100,7 @@ private void Match(DirectoryInfoBase directory, string? parentRelativePath) { if (entity is FileInfoBase fileInfo) { - PatternTestResult result = MatchPatternContexts(fileInfo, (pattern, file) => pattern.Test(file)); + PatternTestResult result = _patternContext.Test(fileInfo); if (result.IsSuccessful) { _files.Add(new FilePatternMatch( @@ -102,7 +113,7 @@ private void Match(DirectoryInfoBase directory, string? parentRelativePath) if (entity is DirectoryInfoBase directoryInfo) { - if (MatchPatternContexts(directoryInfo, (pattern, dir) => pattern.Test(dir))) + if (_patternContext.Test(directoryInfo)) { subDirectories.Add(directoryInfo); } @@ -120,7 +131,7 @@ private void Match(DirectoryInfoBase directory, string? parentRelativePath) } // Request all the including and excluding patterns to pop their status stack. - PopDirectory(); + _patternContext.PopDirectory(); } private void Declare() @@ -129,10 +140,7 @@ private void Declare() _declaredParentPathSegment = false; _declaredWildcardPathSegment = false; - foreach (IPatternContext include in _includePatternContexts) - { - include.Declare(DeclareInclude); - } + _patternContext.Declare(DeclareInclude); } private void DeclareInclude(IPathSegment patternSegment, bool isLastSegment) @@ -169,82 +177,5 @@ internal static string CombinePath(string? left, string right) return $"{left}/{right}"; } } - - // Used to adapt Test(DirectoryInfoBase) for the below overload - private bool MatchPatternContexts(TFileInfoBase fileinfo, Func test) - { - return MatchPatternContexts( - fileinfo, - (ctx, file) => - { - if (test(ctx, file)) - { - return PatternTestResult.Success(stem: string.Empty); - } - else - { - return PatternTestResult.Failed; - } - }).IsSuccessful; - } - - private PatternTestResult MatchPatternContexts(TFileInfoBase fileinfo, Func test) - { - PatternTestResult result = PatternTestResult.Failed; - - // If the given file/directory matches any including pattern, continues to next step. - foreach (IPatternContext context in _includePatternContexts) - { - PatternTestResult localResult = test(context, fileinfo); - if (localResult.IsSuccessful) - { - result = localResult; - break; - } - } - - // If the given file/directory doesn't match any of the including pattern, returns false. - if (!result.IsSuccessful) - { - return PatternTestResult.Failed; - } - - // If the given file/directory matches any excluding pattern, returns false. - foreach (IPatternContext context in _excludePatternContexts) - { - if (test(context, fileinfo).IsSuccessful) - { - return PatternTestResult.Failed; - } - } - - return result; - } - - private void PopDirectory() - { - foreach (IPatternContext context in _excludePatternContexts) - { - context.PopDirectory(); - } - - foreach (IPatternContext context in _includePatternContexts) - { - context.PopDirectory(); - } - } - - private void PushDirectory(DirectoryInfoBase directory) - { - foreach (IPatternContext context in _includePatternContexts) - { - context.PushDirectory(directory); - } - - foreach (IPatternContext context in _excludePatternContexts) - { - context.PushDirectory(directory); - } - } } } diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/CompositePatternContext.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/CompositePatternContext.cs new file mode 100644 index 00000000000..f8a7106a0b4 --- /dev/null +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/CompositePatternContext.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; + +namespace Microsoft.Extensions.FileSystemGlobbing.Internal.PatternContexts +{ + internal abstract class CompositePatternContext : IPatternContext + { + public abstract void Declare(Action onDeclare); + public abstract void PopDirectory(); + public abstract void PushDirectory(DirectoryInfoBase directory); + + protected internal abstract PatternTestResult MatchPatternContexts( + TFileInfoBase fileInfo, + Func test); + + public bool Test(DirectoryInfoBase directory) => + MatchPatternContexts(directory, + static (context, dir) => + context.Test(dir) ? PatternTestResult.Success(stem: string.Empty) : PatternTestResult.Failed).IsSuccessful; + + public PatternTestResult Test(FileInfoBase file) => + MatchPatternContexts(file, static (context, fileInfo) => context.Test(fileInfo)); + } +} diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/IncludesFirstCompositePatternContext.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/IncludesFirstCompositePatternContext.cs new file mode 100644 index 00000000000..5bd375fcddf --- /dev/null +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/IncludesFirstCompositePatternContext.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; + +namespace Microsoft.Extensions.FileSystemGlobbing.Internal.PatternContexts +{ + internal sealed class IncludesFirstCompositePatternContext : CompositePatternContext + { + private readonly IPatternContext[] _includePatternContexts; + private readonly IPatternContext[] _excludePatternContexts; + + internal IncludesFirstCompositePatternContext(IPatternContext[] includePatternContexts, IPatternContext[] excludePatternContexts) + { + _includePatternContexts = includePatternContexts; + _excludePatternContexts = excludePatternContexts; + } + + public override void Declare(Action onDeclare) + { + foreach (IPatternContext include in _includePatternContexts) + { + include.Declare(onDeclare); + } + } + + protected internal override PatternTestResult MatchPatternContexts(TFileInfoBase fileInfo, Func test) + { + PatternTestResult result = PatternTestResult.Failed; + + // If the given file/directory matches any including pattern, continues to next step. + foreach (IPatternContext context in _includePatternContexts) + { + PatternTestResult localResult = test(context, fileInfo); + if (localResult.IsSuccessful) + { + result = localResult; + break; + } + } + + // If the given file/directory doesn't match any of the including pattern, returns false. + if (!result.IsSuccessful) + { + return PatternTestResult.Failed; + } + + // If the given file/directory matches any excluding pattern, returns false. + foreach (IPatternContext context in _excludePatternContexts) + { + if (test(context, fileInfo).IsSuccessful) + { + return PatternTestResult.Failed; + } + } + + return result; + } + + public override void PopDirectory() + { + foreach (IPatternContext context in _excludePatternContexts) + { + context.PopDirectory(); + } + + foreach (IPatternContext context in _includePatternContexts) + { + context.PopDirectory(); + } + } + + public override void PushDirectory(DirectoryInfoBase directory) + { + foreach (IPatternContext context in _includePatternContexts) + { + context.PushDirectory(directory); + } + + foreach (IPatternContext context in _excludePatternContexts) + { + context.PushDirectory(directory); + } + } + } +} diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/PreserveOrderCompositePatternContext.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/PreserveOrderCompositePatternContext.cs new file mode 100644 index 00000000000..79630b8460d --- /dev/null +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Internal/PatternContexts/PreserveOrderCompositePatternContext.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; + +namespace Microsoft.Extensions.FileSystemGlobbing.Internal.PatternContexts +{ + internal sealed class PreserveOrderCompositePatternContext : CompositePatternContext + { + private readonly IncludeOrExcludeValue[] _includeOrExcludePatternContexts; + + internal PreserveOrderCompositePatternContext(IncludeOrExcludeValue[] includeOrExcludePatternContexts) => + _includeOrExcludePatternContexts = includeOrExcludePatternContexts; + + public override void Declare(Action onDeclare) + { + foreach (IncludeOrExcludeValue context in _includeOrExcludePatternContexts) + { + if (context.IsInclude) + { + context.Value.Declare(onDeclare); + } + } + } + + protected internal override PatternTestResult MatchPatternContexts(TFileInfoBase fileInfo, Func test) + { + PatternTestResult result = PatternTestResult.Failed; + + foreach (IncludeOrExcludeValue context in _includeOrExcludePatternContexts) + { + // If the file is currently a match and the pattern is exclude, then test it to determine + // if we should unmatch. And if the file is currently not a match and the pattern is include, + // then test it to determine if we should match. + if (result.IsSuccessful != context.IsInclude) + { + PatternTestResult localResult = test(context.Value, fileInfo); + if (localResult.IsSuccessful) + { + result = context.IsInclude ? localResult : PatternTestResult.Failed; + } + } + } + + return result; + } + + public override void PopDirectory() + { + foreach (IncludeOrExcludeValue context in _includeOrExcludePatternContexts) + { + context.Value.PopDirectory(); + } + } + + public override void PushDirectory(DirectoryInfoBase directory) + { + foreach (IncludeOrExcludeValue context in _includeOrExcludePatternContexts) + { + context.Value.PushDirectory(directory); + } + } + } +} diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Matcher.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Matcher.cs index c95c42d0c8e..61503f7e0e4 100644 --- a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Matcher.cs +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/src/Matcher.cs @@ -96,16 +96,18 @@ namespace Microsoft.Extensions.FileSystemGlobbing /// public class Matcher { - private readonly List _includePatterns = new List(); - private readonly List _excludePatterns = new List(); + private readonly List? _includePatterns; + private readonly List? _excludePatterns; + private readonly List>? _includeOrExcludePatterns; private readonly PatternBuilder _builder; private readonly StringComparison _comparison; + private readonly bool _preserveFilterOrder; /// /// Initializes an instance of using case-insensitive matching /// public Matcher() - : this(StringComparison.OrdinalIgnoreCase) + : this(StringComparison.OrdinalIgnoreCase, false) { } @@ -114,9 +116,33 @@ public Matcher() /// /// The to use public Matcher(StringComparison comparisonType) + : this(comparisonType, false) + { + } + + /// + /// Initializes an instance of using the string comparison method and filter ordering specified + /// + /// The to use + /// + /// if the filters should be applied in the order they were added; + /// if the inclusion filters should be applied before the exclusion filters + /// + public Matcher(StringComparison comparisonType = StringComparison.OrdinalIgnoreCase, bool preserveFilterOrder = false) { _comparison = comparisonType; _builder = new PatternBuilder(comparisonType); + _preserveFilterOrder = preserveFilterOrder; + + if (preserveFilterOrder) + { + _includeOrExcludePatterns = []; + } + else + { + _includePatterns = []; + _excludePatterns = []; + } } /// @@ -133,7 +159,11 @@ public Matcher(StringComparison comparisonType) /// The matcher public virtual Matcher AddInclude(string pattern) { - _includePatterns.Add(_builder.Build(pattern)); + if (_preserveFilterOrder) + _includeOrExcludePatterns!.Add(new IncludeOrExcludeValue { Value = _builder.Build(pattern), IsInclude = true }); + else + _includePatterns!.Add(_builder.Build(pattern)); + return this; } @@ -151,7 +181,11 @@ public virtual Matcher AddInclude(string pattern) /// The matcher public virtual Matcher AddExclude(string pattern) { - _excludePatterns.Add(_builder.Build(pattern)); + if (_preserveFilterOrder) + _includeOrExcludePatterns!.Add(new IncludeOrExcludeValue { Value = _builder.Build(pattern), IsInclude = false }); + else + _excludePatterns!.Add(_builder.Build(pattern)); + return this; } @@ -164,8 +198,9 @@ public virtual PatternMatchingResult Execute(DirectoryInfoBase directoryInfo) { ArgumentNullException.ThrowIfNull(directoryInfo); - var context = new MatcherContext(_includePatterns, _excludePatterns, directoryInfo, _comparison); - return context.Execute(); + return _preserveFilterOrder ? + new MatcherContext(_includeOrExcludePatterns!, directoryInfo, _comparison).Execute() : + new MatcherContext(_includePatterns!, _excludePatterns!, directoryInfo, _comparison).Execute(); } } } diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/OrderedPatternMatchingTests.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/OrderedPatternMatchingTests.cs new file mode 100644 index 00000000000..0cd0ed6ffa4 --- /dev/null +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/OrderedPatternMatchingTests.cs @@ -0,0 +1,210 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; +using Microsoft.Extensions.FileSystemGlobbing.Tests.TestUtility; +using Xunit; + +namespace Microsoft.Extensions.FileSystemGlobbing.Tests +{ + public class OrderedPatternMatchingTests : PatternMatchingTests + { + protected override bool PreserveFilterOrder => true; + + [Fact] + public void MixedPatternSequence_SimpleFilters() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/project/", true) + .Include("A/*") + .Exclude("A/B/*") + .Include("C/*") + .Files( + "A/Program.cs", + "A/B/foo.txt", + "C/README.md" + ) + .Execute(); + + scenario.AssertExact( + "A/Program.cs", + "C/README.md" + ); + } + + [Fact] + public void MixedPatternSequence_DeeplyNestedFilters() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/project/", true) + .Include("**/*.cs") // Include all .cs files + .Exclude("**/obj/**/*") // Exclude all in obj/ + .Include("lib/generated/**/*.cs") // Re-include generated code + .Exclude("**/*Tests.cs") // Exclude unit tests + .Include("tests/manual/*Tests.cs") // Re-include manual test files + .Exclude("legacy/**/*") // Exclude legacy code + .Files( + "Program.cs", + "obj/Temp.cs", + "lib/generated/AutoGen.cs", + "Tests/Unit/MathTests.cs", + "tests/manual/VisualTests.cs", + "legacy/OldStuff.cs", + "README.md" + ) + .Execute(); + + scenario.AssertExact( + "Program.cs", + "lib/generated/AutoGen.cs", + "tests/manual/VisualTests.cs" + ); + } + + [Fact] + public void MixOfPatternsAcrossVariousFileTypes() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/assets/", true) + .Include("**/*") // Include everything + .Exclude("**/*.tmp") // Remove temp files + .Exclude("**/*.bak") // Remove backups + .Include("backup/important/*.bak") // Bring back important backups + .Exclude("temp/**/*") // Remove all in temp dir + .Include("temp/keep/*") // But allow files in temp/keep + .Files( + "image.png", + "video.mp4", + "doc.tmp", + "settings.bak", + "backup/important/settings.bak", + "temp/scratch.txt", + "temp/keep/notes.txt" + ) + .Execute(); + + scenario.AssertExact( + "image.png", + "video.mp4", + "backup/important/settings.bak", + "temp/keep/notes.txt" + ); + } + + [Fact] + public void IncludesThenGlobalExcludeOverrides() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/data/", true) + .Include("**/*.json") + .Include("**/*.csv") + .Include("metrics/*.xml") + .Include("raw/**/*.bin") + .Exclude("**/sensitive/**/*") + .Exclude("**/*.bak") + .Files( + "dataset.csv", + "config.json", + "metrics/stats.xml", + "raw/input/data.bin", + "sensitive/backup/config.bak", + "sensitive/raw/logs.json" + ) + .Execute(); + + scenario.AssertExact( + "dataset.csv", + "config.json", + "metrics/stats.xml", + "raw/input/data.bin" + ); + } + + [Fact] + public void DoubleExcludeExcludesFile() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/data/", true) + .Exclude("**/a.txt") + .Exclude("**/a.txt") + .Files("a.txt") + .Execute(); + + scenario.AssertExact(); + } + + [Fact] + public void DoubleIncludeIncludesFile() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/data/", true) + .Include("**/a.txt") + .Include("**/a.txt") + .Files("a.txt") + .Execute(); + + scenario.AssertExact("a.txt"); + } + + [Fact] + public void IncludeDoubleExcludeExcludesFile() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/data/", true) + .Include("**/a.txt") + .Exclude("**/a.txt") + .Exclude("**/a.txt") + .Files("a.txt") + .Execute(); + + scenario.AssertExact(); + } + + [Fact] + public void ExcludeDoubleIncludeIncludesFile() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/data/", true) + .Exclude("**/a.txt") + .Include("**/a.txt") + .Include("**/a.txt") + .Files("a.txt") + .Execute(); + + scenario.AssertExact("a.txt"); + } + + [Fact] + public void NoFilterExcludes() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/data/", true) + .Files("a.txt") + .Execute(); + + scenario.AssertExact(); + } + + [Fact] + public void PreserveFilterOrderingFalse() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/data/", false) + .Exclude("**/a.txt") + .Include("**/a.txt") + .Files("a.txt") + .Execute(); + + scenario.AssertExact(); + } + + [Fact] + public void ReIncludeSubdir() + { + var scenario = new FileSystemGlobbingTestContext(@"c:/project/", true) + .Include("src/") + .Exclude("src/Internal/") + .Include("src/Internal/PatternContexts/") + .Files( + "src/Internal/PatternContexts/PatternContext.cs", + "src/Internal/PatternSegments/CurrentPathSegment.cs" + ) + .Execute(); + + scenario.AssertExact( + "src/Internal/PatternContexts/PatternContext.cs" + ); + } + } +} diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/PatternMatchingTests.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/PatternMatchingTests.cs index 7b2929316ef..b66dc384c3f 100644 --- a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/PatternMatchingTests.cs +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/PatternMatchingTests.cs @@ -9,11 +9,12 @@ namespace Microsoft.Extensions.FileSystemGlobbing.Tests { public class PatternMatchingTests { + protected virtual bool PreserveFilterOrder => false; + [Fact] public void EmptyCollectionWhenNoFilesPresent() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("alpha.txt") .Execute(); @@ -23,8 +24,7 @@ public void EmptyCollectionWhenNoFilesPresent() [Fact] public void MatchingFileIsFound() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("alpha.txt") .Files("alpha.txt") .Execute(); @@ -35,8 +35,7 @@ public void MatchingFileIsFound() [Fact] public void MismatchedFileIsIgnored() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("alpha.txt") .Files("omega.txt") .Execute(); @@ -47,8 +46,7 @@ public void MismatchedFileIsIgnored() [Fact] public void FolderNamesAreTraversed() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("beta/alpha.txt") .Files("beta/alpha.txt") .Execute(); @@ -64,8 +62,7 @@ public void FolderNamesAreTraversed() [InlineData(@"\beta\alpha.txt", @"beta\alpha.txt")] public void SlashPolarityIsIgnored(string includePattern, string filePath) { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include(includePattern) .Files("one/two.txt", filePath, "three/four.txt") .Execute(); @@ -83,8 +80,7 @@ public void SlashPolarityIsIgnored(string includePattern, string filePath) [InlineData(@"b*et*x", new string[0])] public void PatternMatchingWorks(string includePattern, string[] matchesExpected) { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include(includePattern) .Files("alpha.txt", "beta.txt", "gamma.dat") .Execute(); @@ -101,8 +97,7 @@ public void PatternMatchingWorks(string includePattern, string[] matchesExpected [InlineData(@"*45*56", new string[0])] public void PatternBeginAndEndCantOverlap(string includePattern, string[] matchesExpected) { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include(includePattern) .Files("12345678") .Execute(); @@ -122,8 +117,7 @@ public void PatternBeginAndEndCantOverlap(string includePattern, string[] matche [InlineData(@"/*.*/*", new[] { "alpha/hello.txt", "beta/hello.txt", "gamma/hello.txt" })] public void PatternMatchingWorksInFolders(string includePattern, string[] matchesExpected) { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include(includePattern) .Files("alpha/hello.txt", "beta/hello.txt", "gamma/hello.txt") .Execute(); @@ -143,8 +137,7 @@ public void PatternMatchingWorksInFolders(string includePattern, string[] matche [InlineData(@"./*mm*/*", new string[] { "gamma/hello.txt" })] public void PatternMatchingCurrent(string includePattern, string[] matchesExpected) { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include(includePattern) .Files("alpha/hello.txt", "beta/hello.txt", "gamma/hello.txt") .Execute(); @@ -155,8 +148,7 @@ public void PatternMatchingCurrent(string includePattern, string[] matchesExpect [Fact] public void StarDotStarIsSameAsStar() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("*.*") .Files("alpha.txt", "alpha.", ".txt", ".", "alpha", "txt") .Execute(); @@ -167,8 +159,7 @@ public void StarDotStarIsSameAsStar() [Fact] public void IncompletePatternsDoNotInclude() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("*/*.txt") .Files("one/x.txt", "two/x.txt", "x.txt") .Execute(); @@ -179,8 +170,7 @@ public void IncompletePatternsDoNotInclude() [Fact] public void IncompletePatternsDoNotExclude() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("*/*.txt") .Exclude("one/hello.txt") .Files("one/x.txt", "two/x.txt") @@ -192,8 +182,7 @@ public void IncompletePatternsDoNotExclude() [Fact] public void TrailingRecursiveWildcardMatchesAllFiles() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("one/**") .Files("one/x.txt", "two/x.txt", "one/x/y.txt") .Execute(); @@ -204,8 +193,7 @@ public void TrailingRecursiveWildcardMatchesAllFiles() [Fact] public void LeadingRecursiveWildcardMatchesAllLeadingPaths() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("**/*.cs") .Files("one/x.cs", "two/x.cs", "one/two/x.cs", "x.cs") .Files("one/x.txt", "two/x.txt", "one/two/x.txt", "x.txt") @@ -217,8 +205,7 @@ public void LeadingRecursiveWildcardMatchesAllLeadingPaths() [Fact] public void InnerRecursiveWildcardMuseStartWithAndEndWith() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("one/**/*.cs") .Files("one/x.cs", "two/x.cs", "one/two/x.cs", "x.cs") .Files("one/x.txt", "two/x.txt", "one/two/x.txt", "x.txt") @@ -231,8 +218,7 @@ public void InnerRecursiveWildcardMuseStartWithAndEndWith() [Fact] public void ExcludeMayEndInDirectoryName() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("*.cs", "*/*.cs", "*/*/*.cs") .Exclude("bin", "one/two") .Files("one/x.cs", "two/x.cs", "one/two/x.cs", "x.cs", "bin/x.cs", "bin/two/x.cs") @@ -245,8 +231,7 @@ public void ExcludeMayEndInDirectoryName() [Fact] public void RecursiveWildcardSurroundingContainsWith() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("**/x/**") .Files("x/1", "1/x/2", "1/x", "x", "1", "1/2") .Execute(); @@ -258,8 +243,7 @@ public void RecursiveWildcardSurroundingContainsWith() [Fact] public void SequentialFoldersMayBeRequired() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("a/b/**/1/2/**/2/3/**") .Files("1/2/2/3/x", "1/2/3/y", "a/1/2/4/2/3/b", "a/2/3/1/2/b") .Files("a/b/1/2/2/3/x", "a/b/1/2/3/y", "a/b/a/1/2/4/2/3/b", "a/b/a/2/3/1/2/b") @@ -271,8 +255,7 @@ public void SequentialFoldersMayBeRequired() [Fact] public void RecursiveAloneIncludesEverything() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("**") .Files("1/2/2/3/x", "1/2/3/y") .Execute(); @@ -283,8 +266,7 @@ public void RecursiveAloneIncludesEverything() [Fact] public void ExcludeCanHaveSurroundingRecursiveWildcards() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("**") .Exclude("**/x/**") .Files("x/1", "1/x/2", "1/x", "x", "1", "1/2") @@ -296,8 +278,7 @@ public void ExcludeCanHaveSurroundingRecursiveWildcards() [Fact] public void LeadingDotDotCanComeThroughPattern() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("*.cs") .Include("../2/*.cs") .Files("1/x.cs", "1/x.txt", "2/x.cs", "2/x.txt") @@ -310,8 +291,7 @@ public void LeadingDotDotCanComeThroughPattern() [Fact] public void LeadingDotDotWithRecursiveCanComeThroughPattern() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("*.cs") .Include("../2/**/*.cs") .Files("1/x.cs", "1/x.txt", "2/x.cs", "2/x.txt", "2/3/x.cs", "2/3/4/z.cs", "2/3/x.txt") @@ -324,8 +304,7 @@ public void LeadingDotDotWithRecursiveCanComeThroughPattern() [Fact] public void ExcludeFolderRecursively() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("*.*") .Include("../sibling/**/*.*") .Exclude("../sibling/exc/**/*.*") @@ -340,8 +319,7 @@ public void ExcludeFolderRecursively() [Fact] public void ExcludeFolderByName() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("*.*") .Include("../sibling/**/*.*") .Exclude("../sibling/exc/") @@ -356,8 +334,7 @@ public void ExcludeFolderByName() [Fact] public void MultipleRecursiveWildcardStemMatch() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("sub/**/bar/**/*.txt") .Files("root.txt", "sub/one.txt", "sub/two.txt", "sub/sub2/bar/baz/three.txt", "sub/sub3/sub4/bar/three.txt") .Execute(); @@ -372,8 +349,7 @@ public void MultipleRecursiveWildcardStemMatch() [Fact] public void RecursiveWildcardStemMatch() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("sub/**/*.txt") .Files("root.txt", "sub/one.txt", "sub/two.txt", "sub/sub2/three.txt") .Execute(); @@ -389,8 +365,7 @@ public void RecursiveWildcardStemMatch() [Fact] public void WildcardMidSegmentMatch() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("sub/w*.txt") .Files("root.txt", "sub/woah.txt", "sub/wow.txt", "sub/blah.txt") .Execute(); @@ -405,8 +380,7 @@ public void WildcardMidSegmentMatch() [Fact] public void StemMatchOnExactFile() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("sub/sub/three.txt") .Files("root.txt", "sub/one.txt", "sub/two.txt", "sub/sub/three.txt") .Execute(); @@ -420,8 +394,7 @@ public void StemMatchOnExactFile() [Fact] public void SimpleStemMatching() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("sub/*") .Files("root.txt", "sub/one.txt", "sub/two.txt", "sub/sub/three.txt") .Execute(); @@ -436,8 +409,7 @@ public void SimpleStemMatching() [Fact] public void StemMatchingWithFileExtension() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("sub/*.txt") .Files("root.txt", "sub/one.txt", "sub/two.txt", "sub/three.dat") .Execute(); @@ -452,8 +424,7 @@ public void StemMatchingWithFileExtension() [Fact] public void StemMatchingWithParentDir() { - var matcher = new Matcher(); - var scenario = new FileSystemGlobbingTestContext(@"c:\files\", matcher) + var scenario = new FileSystemGlobbingTestContext(@"c:\files\", PreserveFilterOrder) .Include("../files/sub/*.txt") .Files("root.txt", "sub/one.txt", "sub/two.txt", "sub/three.dat") .Execute(); diff --git a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/TestUtility/FileSystemGlobbingTestContext.cs b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/TestUtility/FileSystemGlobbingTestContext.cs index 842c3af6c44..9907bf8a2a0 100644 --- a/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/TestUtility/FileSystemGlobbingTestContext.cs +++ b/src/runtime/src/libraries/Microsoft.Extensions.FileSystemGlobbing/tests/TestUtility/FileSystemGlobbingTestContext.cs @@ -16,11 +16,11 @@ internal class FileSystemGlobbingTestContext public PatternMatchingResult Result { get; private set; } - public FileSystemGlobbingTestContext(string basePath, Matcher matcher) + public FileSystemGlobbingTestContext(string basePath, bool preserveFilterOrder) { _basePath = basePath; _recorder = new FileSystemOperationRecorder(); - _patternMatching = matcher; + _patternMatching = new Matcher(preserveFilterOrder: preserveFilterOrder); _directoryInfo = new MockDirectoryInfo( recorder: _recorder, diff --git a/src/runtime/src/libraries/System.Collections/src/System/Collections/BitArray.cs b/src/runtime/src/libraries/System.Collections/src/System/Collections/BitArray.cs index 80491afd598..fa515728e08 100644 --- a/src/runtime/src/libraries/System.Collections/src/System/Collections/BitArray.cs +++ b/src/runtime/src/libraries/System.Collections/src/System/Collections/BitArray.cs @@ -837,7 +837,7 @@ public unsafe void CopyTo(Array array, int index) Vector128 lowerShuffleMask_CopyToBoolArray = Vector128.Create(0, 0x01010101_01010101).AsByte(); Vector128 upperShuffleMask_CopyToBoolArray = Vector128.Create(0x02020202_02020202, 0x03030303_03030303).AsByte(); - if (Avx512F.IsSupported && (uint)m_length >= Vector512.Count) + if (Avx512BW.IsSupported && (uint)m_length >= Vector512.Count) { Vector256 upperShuffleMask_CopyToBoolArray256 = Vector256.Create(0x04040404_04040404, 0x05050505_05050505, 0x06060606_06060606, 0x07070707_07070707).AsByte(); diff --git a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs index 3a2a416704e..89806cb9882 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics; + namespace System.Formats.Tar { /// @@ -34,10 +36,7 @@ internal GnuTarEntry(TarHeader header, TarReader readerOfOrigin) /// is not supported in the specified format. public GnuTarEntry(TarEntryType entryType, string entryName) : base(entryType, entryName, TarEntryFormat.Gnu, isGea: false) - { - _header._aTime = default; - _header._cTime = default; - } + { } /// /// Initializes a new instance by converting the specified entry into the GNU format. @@ -51,21 +50,17 @@ public GnuTarEntry(TarEntryType entryType, string entryName) public GnuTarEntry(TarEntry other) : base(other, TarEntryFormat.Gnu) { + // Some tools don't accept Gnu entries that have an atime/ctime. + // We only copy atime/ctime for round-tripping between GnuTarEntries and clear it for other formats. if (other is GnuTarEntry gnuOther) { _header._aTime = gnuOther.AccessTime; _header._cTime = gnuOther.ChangeTime; - _header._gnuUnusedBytes = other._header._gnuUnusedBytes; } else { - // 'other' was V7, Ustar (those formats do not have atime or ctime), - // or even PAX (which could contain atime and ctime in the ExtendedAttributes), but - // to avoid creating a GNU entry that might be incompatible with other tools, - // we avoid setting the atime and ctime fields. The user would have to set them manually - // if they are really needed. - _header._aTime = default; - _header._cTime = default; + Debug.Assert(_header._aTime == default); + Debug.Assert(_header._cTime == default); } } diff --git a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PaxGlobalExtendedAttributesTarEntry.cs b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PaxGlobalExtendedAttributesTarEntry.cs index 94ba60167f7..9dc572f8c1c 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PaxGlobalExtendedAttributesTarEntry.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PaxGlobalExtendedAttributesTarEntry.cs @@ -28,7 +28,7 @@ public PaxGlobalExtendedAttributesTarEntry(IEnumerable diff --git a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PaxTarEntry.cs b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PaxTarEntry.cs index a062993fef0..30397b03b27 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PaxTarEntry.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PaxTarEntry.cs @@ -21,7 +21,7 @@ internal PaxTarEntry(TarHeader header, TarReader readerOfOrigin) } /// - /// Initializes a new instance with the specified entry type, entry name, and the default extended attributes. + /// Initializes a new instance with the specified entry type and entry name. /// /// The type of the entry. /// A string with the path and file name of this entry. @@ -30,20 +30,7 @@ internal PaxTarEntry(TarHeader header, TarReader readerOfOrigin) /// In all platforms: , , , . /// In Unix platforms only: , and . /// - /// Use the constructor to include additional extended attributes when creating the entry. - /// The following entries are always found in the Extended Attributes dictionary of any PAX entry: - /// - /// Modification time, under the name mtime, as a number. - /// Access time, under the name atime, as a number. - /// Change time, under the name ctime, as a number. - /// Path, under the name path, as a string. - /// - /// The following entries are only found in the Extended Attributes dictionary of a PAX entry if certain conditions are met: - /// - /// Group name, under the name gname, as a string, if it is larger than 32 bytes. - /// User name, under the name uname, as a string, if it is larger than 32 bytes. - /// File length, under the name size, as an , if the string representation of the number is larger than 12 bytes. - /// + /// Use the constructor to include extended attributes when creating the entry. /// /// is . /// is empty. @@ -53,13 +40,10 @@ public PaxTarEntry(TarEntryType entryType, string entryName) : base(entryType, entryName, TarEntryFormat.Pax, isGea: false) { _header._prefix = string.Empty; - - Debug.Assert(_header._mTime != default); - AddNewAccessAndChangeTimestampsIfNotExist(useMTime: true); } /// - /// Initializes a new instance with the specified entry type, entry name and Extended Attributes enumeration. + /// Initializes a new instance with the specified entry type, entry name and extended attributes. /// /// The type of the entry. /// A string with the path and file name of this entry. @@ -69,19 +53,11 @@ public PaxTarEntry(TarEntryType entryType, string entryName) /// In all platforms: , , , . /// In Unix platforms only: , and . /// - /// The specified get appended to the default attributes, unless the specified enumeration overrides any of them. - /// The following entries are always found in the Extended Attributes dictionary of any PAX entry: + /// The specified are additional attributes to be used for the entry. + /// It may include PAX attributes like: /// - /// Modification time, under the name mtime, as a number. /// Access time, under the name atime, as a number. /// Change time, under the name ctime, as a number. - /// Path, under the name path, as a string. - /// - /// The following entries are only found in the Extended Attributes dictionary of a PAX entry if certain conditions are met: - /// - /// Group name, under the name gname, as a string, if it is larger than 32 bytes. - /// User name, under the name uname, as a string, if it is larger than 32 bytes. - /// File length, under the name size, as an , if the string representation of the number is larger than 12 bytes. /// /// /// or is . @@ -94,10 +70,7 @@ public PaxTarEntry(TarEntryType entryType, string entryName, IEnumerable @@ -119,72 +92,39 @@ public PaxTarEntry(TarEntry other) if (other is PaxTarEntry paxOther) { - _header.InitializeExtendedAttributesWithExisting(paxOther.ExtendedAttributes); + _header.AddExtendedAttributes(paxOther.ExtendedAttributes); } - else + else if (other is GnuTarEntry gnuOther) { - if (other is GnuTarEntry gnuOther) + if (gnuOther.AccessTime != default) + { + _header.ExtendedAttributes[TarHeader.PaxEaATime] = TarHelpers.GetTimestampStringFromDateTimeOffset(gnuOther.AccessTime); + } + if (gnuOther.ChangeTime != default) { - if (gnuOther.AccessTime != default) - { - _header.ExtendedAttributes[TarHeader.PaxEaATime] = TarHelpers.GetTimestampStringFromDateTimeOffset(gnuOther.AccessTime); - } - if (gnuOther.ChangeTime != default) - { - _header.ExtendedAttributes[TarHeader.PaxEaCTime] = TarHelpers.GetTimestampStringFromDateTimeOffset(gnuOther.ChangeTime); - } + _header.ExtendedAttributes[TarHeader.PaxEaCTime] = TarHelpers.GetTimestampStringFromDateTimeOffset(gnuOther.ChangeTime); } } - - AddNewAccessAndChangeTimestampsIfNotExist(useMTime: false); } /// /// Returns the extended attributes for this entry. /// - /// The extended attributes are specified when constructing an entry. Use to append your own enumeration of extended attributes to the current entry on top of the default ones. Use to only use the default extended attributes. - /// The following entries are always found in the Extended Attributes dictionary of any PAX entry: + /// The extended attributes are specified when constructing an entry and updated with additional attributes when the entry is written. Use to append custom extended attributes. + /// The following common PAX attributes may be included: /// /// Modification time, under the name mtime, as a number. /// Access time, under the name atime, as a number. /// Change time, under the name ctime, as a number. /// Path, under the name path, as a string. - /// - /// The following entries are only found in the Extended Attributes dictionary of a PAX entry if certain conditions are met: - /// - /// Group name, under the name gname, as a string, if it is larger than 32 bytes. - /// User name, under the name uname, as a string, if it is larger than 32 bytes. - /// File length, under the name size, as an , if the string representation of the number is larger than 12 bytes. + /// Group name, under the name gname, as a string. + /// User name, under the name uname, as a string. + /// File length, under the name size, as an . /// /// public IReadOnlyDictionary ExtendedAttributes => _readOnlyExtendedAttributes ??= _header.ExtendedAttributes.AsReadOnly(); // Determines if the current instance's entry type supports setting a data stream. internal override bool IsDataStreamSetterSupported() => EntryType == TarEntryType.RegularFile; - - // Checks if the extended attributes dictionary contains 'atime' and 'ctime'. - // If any of them is not found, it is added with the value of either the current entry's 'mtime', - // or 'DateTimeOffset.UtcNow', depending on the value of 'useMTime'. - private void AddNewAccessAndChangeTimestampsIfNotExist(bool useMTime) - { - Debug.Assert(!useMTime || (useMTime && _header._mTime != default)); - bool containsATime = _header.ExtendedAttributes.ContainsKey(TarHeader.PaxEaATime); - bool containsCTime = _header.ExtendedAttributes.ContainsKey(TarHeader.PaxEaCTime); - - if (!containsATime || !containsCTime) - { - string secondsFromEpochString = TarHelpers.GetTimestampStringFromDateTimeOffset(useMTime ? _header._mTime : DateTimeOffset.UtcNow); - - if (!containsATime) - { - _header.ExtendedAttributes[TarHeader.PaxEaATime] = secondsFromEpochString; - } - - if (!containsCTime) - { - _header.ExtendedAttributes[TarHeader.PaxEaCTime] = secondsFromEpochString; - } - } - } } } diff --git a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs index e15d7a1add2..37570d0d33a 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs @@ -15,8 +15,6 @@ namespace System.Formats.Tar // Reads the header attributes from a tar archive entry. internal sealed partial class TarHeader { - private readonly byte[] ArrayOf12NullBytes = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; - // Attempts to retrieve the next header from the specified tar archive stream. // Throws if end of stream is reached or if any data type conversion fails. // Returns a valid TarHeader object if the attributes were read successfully, null otherwise. @@ -106,7 +104,7 @@ internal void ReplaceNormalAttributesWithExtended(Dictionary? di return; } - InitializeExtendedAttributesWithExisting(dictionaryFromExtendedAttributesHeader); + AddExtendedAttributes(dictionaryFromExtendedAttributesHeader); // Find all the extended attributes with known names and save them in the expected standard attribute. @@ -388,7 +386,7 @@ private async Task ProcessDataBlockAsync(Stream archiveStream, bool copyData, Ca TarHeader header = new(initialFormat, name: TarHelpers.GetTrimmedUtf8String(buffer.Slice(FieldLocations.Name, FieldLengths.Name)), mode: TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.Mode, FieldLengths.Mode)), - mTime: TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.MTime, FieldLengths.MTime))), + mTime: ParseAsTimestamp(buffer.Slice(FieldLocations.MTime, FieldLengths.MTime)), typeFlag: (TarEntryType)buffer[FieldLocations.TypeFlag]) { _checksum = checksum, @@ -538,21 +536,24 @@ private void ReadPosixAndGnuSharedAttributes(ReadOnlySpan buffer) // Throws if any conversion fails. private void ReadGnuAttributes(ReadOnlySpan buffer) { - // Convert byte arrays - ReadOnlySpan aTimeBuffer = buffer.Slice(FieldLocations.ATime, FieldLengths.ATime); - if (!aTimeBuffer.SequenceEqual(ArrayOf12NullBytes)) // null values are ignored - { - long aTime = TarHelpers.ParseNumeric(aTimeBuffer); - _aTime = TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(aTime); - } - ReadOnlySpan cTimeBuffer = buffer.Slice(FieldLocations.CTime, FieldLengths.CTime); - if (!cTimeBuffer.SequenceEqual(ArrayOf12NullBytes)) // An all nulls buffer is interpreted as MinValue + _aTime = ParseAsTimestamp(buffer.Slice(FieldLocations.ATime, FieldLengths.ATime)); + + _cTime = ParseAsTimestamp(buffer.Slice(FieldLocations.CTime, FieldLengths.CTime)); + + // TODO: Read the bytes of the currently unsupported GNU fields, in case user wants to write this entry into another GNU archive, they need to be preserved. https://github.com/dotnet/runtime/issues/68230 + } + + private static DateTimeOffset ParseAsTimestamp(ReadOnlySpan buffer) + { + // When all bytes are zero, the timestamp is not initialized, and we map it to default. + bool allZeros = !buffer.ContainsAnyExcept((byte)0); + if (allZeros) { - long cTime = TarHelpers.ParseNumeric(cTimeBuffer); - _cTime = TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(cTime); + return default(DateTimeOffset); } - // TODO: Read the bytes of the currently unsupported GNU fields, in case user wants to write this entry into another GNU archive, they need to be preserved. https://github.com/dotnet/runtime/issues/68230 + long time = TarHelpers.ParseNumeric(buffer); + return TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(time); } // Reads the ustar prefix attribute. diff --git a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs index 421cf210efd..df33b14d35a 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs @@ -790,19 +790,8 @@ private int WriteGnuFields(Span buffer) if (_typeFlag is not TarEntryType.LongLink and not TarEntryType.LongPath) { - if (_aTime != default) - { - checksum += WriteAsTimestamp(_aTime, buffer.Slice(FieldLocations.ATime, FieldLengths.ATime)); - } - if (_cTime != default) - { - checksum += WriteAsTimestamp(_cTime, buffer.Slice(FieldLocations.CTime, FieldLengths.CTime)); - } - } - - if (_gnuUnusedBytes != null) - { - checksum += WriteLeftAlignedBytesAndGetChecksum(_gnuUnusedBytes, buffer.Slice(FieldLocations.GnuUnused, FieldLengths.AllGnuUnused)); + checksum += WriteAsTimestamp(_aTime, buffer.Slice(FieldLocations.ATime, FieldLengths.ATime)); + checksum += WriteAsTimestamp(_cTime, buffer.Slice(FieldLocations.CTime, FieldLengths.CTime)); } return checksum; @@ -1179,6 +1168,12 @@ private static int FormatOctal(long value, Span destination) // Writes the specified DateTimeOffset's Unix time seconds, and returns its checksum. private int WriteAsTimestamp(DateTimeOffset timestamp, Span destination) { + // For 'default' we leave the buffer zero-ed to indicate: "no timestamp". + if (timestamp == default) + { + return 0; + } + long unixTimeSeconds = timestamp.ToUnixTimeSeconds(); return FormatNumeric(unixTimeSeconds, destination); } diff --git a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.cs b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.cs index 9306ef1997c..35da5b566ac 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.cs @@ -82,10 +82,6 @@ internal sealed partial class TarHeader internal DateTimeOffset _aTime; internal DateTimeOffset _cTime; - // If the archive is GNU and the offset, longnames, unused, sparse, isextended and realsize - // fields have data, we store it to avoid data loss, but we don't yet expose it publicly. - internal byte[]? _gnuUnusedBytes; - // Constructor called when creating an entry with default common fields. internal TarHeader(TarEntryFormat format, string name = "", int mode = 0, DateTimeOffset mTime = default, TarEntryType typeFlag = TarEntryType.RegularFile) { @@ -112,7 +108,7 @@ internal TarHeader(TarEntryFormat format, TarEntryType typeFlag, TarHeader other _dataStream = other._dataStream; } - internal void InitializeExtendedAttributesWithExisting(IEnumerable> existing) + internal void AddExtendedAttributes(IEnumerable> existing) { Debug.Assert(_ea == null); Debug.Assert(existing != null); diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/GnuTarEntry.Conversion.Tests.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/GnuTarEntry.Conversion.Tests.cs index c1ba5c62bc6..ee5c1e31bcb 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/GnuTarEntry.Conversion.Tests.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/GnuTarEntry.Conversion.Tests.cs @@ -13,6 +13,8 @@ namespace System.Formats.Tar.Tests { public class GnuTarEntry_Conversion_Tests : TarTestsConversionBase { + protected override TarEntryFormat FormatUnderTest => TarEntryFormat.Gnu; + [Fact] public void Constructor_ConversionFromV7_RegularFile() => TestConstructionConversion(TarEntryType.RegularFile, TarEntryFormat.V7, TarEntryFormat.Gnu); diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/PaxTarEntry.Conversion.Tests.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/PaxTarEntry.Conversion.Tests.cs index eb80726dbe0..ba4eec334d8 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/PaxTarEntry.Conversion.Tests.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/PaxTarEntry.Conversion.Tests.cs @@ -13,6 +13,8 @@ namespace System.Formats.Tar.Tests { public class PaxTarEntry_Conversion_Tests : TarTestsConversionBase { + protected override TarEntryFormat FormatUnderTest => TarEntryFormat.Pax; + [Fact] public void Constructor_ConversionFromV7_RegularFile() => TestConstructionConversion(TarEntryType.RegularFile, TarEntryFormat.V7, TarEntryFormat.Pax); @@ -92,8 +94,8 @@ public void Constructor_ConversionFromV7_Write(TarEntryFormat originalEntryForma dataStream.Position = 0; originalEntry.DataStream = dataStream; - DateTimeOffset expectedATime = default; - DateTimeOffset expectedCTime = default; + DateTimeOffset? expectedATime = null; + DateTimeOffset? expectedCTime = null; if (originalEntry is GnuTarEntry gnuEntry) { @@ -108,11 +110,9 @@ public void Constructor_ConversionFromV7_Write(TarEntryFormat originalEntryForma } else if (originalEntry is PaxTarEntry paxEntry) { - expectedATime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaATime); - expectedCTime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaCTime); + expectedATime = TryGetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaATime); + expectedCTime = TryGetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(paxEntry.ModificationTime, expectedATime); - Assert.Equal(paxEntry.ModificationTime, expectedCTime); // Can't change them, it's a read-only dictionary } @@ -140,19 +140,11 @@ public void Constructor_ConversionFromV7_Write(TarEntryFormat originalEntryForma Assert.Equal(contents, streamReader.ReadLine()); } - // atime and ctime should've been added automatically in the conversion constructor - // and should not be equal to the value of mtime, which was set on the original entry constructor - - Assert.Contains(PaxEaATime, paxEntry.ExtendedAttributes); - Assert.Contains(PaxEaCTime, paxEntry.ExtendedAttributes); - DateTimeOffset atime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes[PaxEaATime]); - DateTimeOffset ctime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes[PaxEaCTime]); + DateTimeOffset? atime = TryGetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaATime); + DateTimeOffset? ctime = TryGetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaCTime); - if (originalEntryFormat is TarEntryFormat.Pax or TarEntryFormat.Gnu) - { - Assert.Equal(expectedATime, atime); - Assert.Equal(expectedCTime, ctime); - } + Assert.Equal(expectedATime, atime); + Assert.Equal(expectedCTime, ctime); } } diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/TarEntry.Conversion.Tests.Base.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/TarEntry.Conversion.Tests.Base.cs index fef2da357e8..f35052d453a 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/TarEntry.Conversion.Tests.Base.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/TarEntry.Conversion.Tests.Base.cs @@ -6,14 +6,17 @@ namespace System.Formats.Tar.Tests { - public class TarTestsConversionBase : TarTestsBase + public abstract class TarTestsConversionBase : TarTestsBase { + protected abstract TarEntryFormat FormatUnderTest { get; } + private readonly TimeSpan _oneSecond = TimeSpan.FromSeconds(1); protected void TestConstructionConversion( TarEntryType originalEntryType, TarEntryFormat firstFormat, - TarEntryFormat formatToConvert) + TarEntryFormat formatToConvert, + bool setATimeCTime = false) { DateTimeOffset now = DateTimeOffset.UtcNow; @@ -21,7 +24,7 @@ protected void TestConstructionConversion( TarEntryType actualEntryType = GetTarEntryTypeForTarEntryFormat(originalEntryType, firstFormat); - TarEntry firstEntry = GetFirstEntry(dataStream, actualEntryType, firstFormat); + TarEntry firstEntry = GetFirstEntry(dataStream, actualEntryType, firstFormat, setATimeCTime); TarEntry otherEntry = ConvertAndVerifyEntry(firstEntry, originalEntryType, formatToConvert, now); } @@ -42,9 +45,9 @@ protected void TestConstructionConversionBackAndForth( ConvertAndVerifyEntry(otherEntry, firstAndLastEntryType, firstAndLastFormat, secondNow); // Convert back to original format } - private TarEntry GetFirstEntry(MemoryStream dataStream, TarEntryType entryType, TarEntryFormat format) + private TarEntry GetFirstEntry(MemoryStream dataStream, TarEntryType entryType, TarEntryFormat format, bool setATimeCTime = false) { - TarEntry firstEntry = InvokeTarEntryCreationConstructor(format, entryType, "file.txt"); + TarEntry firstEntry = InvokeTarEntryCreationConstructor(format, entryType, "file.txt", setATimeCTime); firstEntry.Gid = TestGid; firstEntry.Uid = TestUid; @@ -66,21 +69,6 @@ private TarEntry GetFirstEntry(MemoryStream dataStream, TarEntryType entryType, posixTarEntry.DeviceMinor = entryType is TarEntryType.BlockDevice ? TestBlockDeviceMinor : TestCharacterDeviceMinor; } - if (format is TarEntryFormat.Pax) - { - PaxTarEntry paxEntry = firstEntry as PaxTarEntry; - Assert.Contains("atime", paxEntry.ExtendedAttributes); - Assert.Contains("ctime", paxEntry.ExtendedAttributes); - Assert.Equal(firstEntry.ModificationTime, GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, "atime")); - Assert.Equal(firstEntry.ModificationTime, GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, "ctime")); - } - else if (format is TarEntryFormat.Gnu) - { - GnuTarEntry gnuEntry = firstEntry as GnuTarEntry; - Assert.Equal(default, gnuEntry.AccessTime); - Assert.Equal(default, gnuEntry.ChangeTime); - } - return firstEntry; } @@ -115,100 +103,72 @@ private TarEntry ConvertAndVerifyEntry(TarEntry originalEntry, TarEntryType entr Assert.Equal(originalPosixTarEntry.DeviceMinor, convertedPosixTarEntry.DeviceMinor); } + // 'null' indicates the originalEntry did not include a timestamp. + GetTimestampsFromEntry(originalEntry, out DateTimeOffset? expectedATime, out DateTimeOffset? expectedCTime); + // Converting to Gnu only preserves timestamps when the original is also Gnu. + if (formatToConvert == TarEntryFormat.Gnu && originalEntry.Format != TarEntryFormat.Gnu) + { + expectedATime = null; + expectedCTime = null; + } + if (formatToConvert is TarEntryFormat.Pax) { PaxTarEntry paxEntry = convertedEntry as PaxTarEntry; - if (originalEntry.Format is TarEntryFormat.Gnu) - { - GnuTarEntry gnuEntry = originalEntry as GnuTarEntry; - - DateTimeOffset expectedATime = gnuEntry.AccessTime; - DateTimeOffset expectedCTime = gnuEntry.ChangeTime; - - DateTimeOffset actualAccessTime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaATime); - DateTimeOffset actualChangeTime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaCTime); - - if (expectedATime == default) - { - AssertExtensions.GreaterThanOrEqualTo(actualAccessTime, paxEntry.ModificationTime); - } - else - { - expectedATime = expectedATime - _oneSecond; - AssertExtensions.GreaterThanOrEqualTo(expectedATime, actualAccessTime); - } - - if (expectedCTime == default) - { - AssertExtensions.GreaterThanOrEqualTo(actualChangeTime, paxEntry.ModificationTime); - } - else - { - expectedCTime = expectedCTime - _oneSecond; - AssertExtensions.GreaterThanOrEqualTo(expectedCTime, actualChangeTime); - } - } - else if (originalEntry.Format is TarEntryFormat.Pax) - { - PaxTarEntry originalPaxEntry = originalEntry as PaxTarEntry; - - DateTimeOffset expectedATime = GetDateTimeOffsetFromTimestampString(originalPaxEntry.ExtendedAttributes, PaxEaATime) - _oneSecond; - DateTimeOffset expectedCTime = GetDateTimeOffsetFromTimestampString(originalPaxEntry.ExtendedAttributes, PaxEaCTime) - _oneSecond; - - DateTimeOffset actualAccessTime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaATime); - DateTimeOffset actualChangeTime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaCTime); - - AssertExtensions.GreaterThanOrEqualTo(actualAccessTime, expectedATime); - AssertExtensions.GreaterThanOrEqualTo(actualChangeTime, expectedCTime); - } - else if (originalEntry.Format is TarEntryFormat.Ustar or TarEntryFormat.V7) - { - DateTimeOffset actualAccessTime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaATime); - DateTimeOffset actualChangeTime = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaCTime); - - AssertExtensions.GreaterThanOrEqualTo(actualAccessTime, initialNow); - AssertExtensions.GreaterThanOrEqualTo(actualChangeTime, initialNow); - } + DateTimeOffset? actualAccessTime = TryGetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaATime); + DateTimeOffset? actualChangeTime = TryGetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, PaxEaCTime); + Assert.Equal(expectedATime, actualAccessTime); + Assert.Equal(expectedCTime, actualChangeTime); } - - if (formatToConvert is TarEntryFormat.Gnu) + else if (formatToConvert is TarEntryFormat.Gnu) { GnuTarEntry gnuEntry = convertedEntry as GnuTarEntry; - if (originalEntry.Format is TarEntryFormat.Pax or TarEntryFormat.Gnu) - { - GetExpectedTimestampsFromOriginalPaxOrGnu(originalEntry, formatToConvert, out DateTimeOffset expectedATime, out DateTimeOffset expectedCTime); - AssertExtensions.GreaterThanOrEqualTo(gnuEntry.AccessTime, expectedATime); - AssertExtensions.GreaterThanOrEqualTo(gnuEntry.ChangeTime, expectedCTime); - } - else if (originalEntry.Format is TarEntryFormat.Ustar or TarEntryFormat.V7) - { - Assert.Equal(default, gnuEntry.AccessTime); - Assert.Equal(default, gnuEntry.ChangeTime); - } + Assert.Equal(expectedATime ?? default, gnuEntry.AccessTime); + Assert.Equal(expectedCTime ?? default, gnuEntry.ChangeTime); } return convertedEntry; } - private void GetExpectedTimestampsFromOriginalPaxOrGnu(TarEntry originalEntry, TarEntryFormat formatToConvert, out DateTimeOffset expectedATime, out DateTimeOffset expectedCTime) + private void GetTimestampsFromEntry(TarEntry originalEntry, out DateTimeOffset? expectedATime, out DateTimeOffset? expectedCTime) { - Assert.True(originalEntry.Format is TarEntryFormat.Gnu or TarEntryFormat.Pax); if (originalEntry.Format is TarEntryFormat.Pax) { PaxTarEntry originalPaxEntry = originalEntry as PaxTarEntry; - Assert.Contains("atime", originalPaxEntry.ExtendedAttributes); // We are verifying that the original had an atime and ctime set - Assert.Contains("ctime", originalPaxEntry.ExtendedAttributes); // and that when converting to GNU we are _not_ preserving them - // And that instead, we are setting them to MinValue - expectedATime = formatToConvert is TarEntryFormat.Gnu ? default : GetDateTimeOffsetFromTimestampString(originalPaxEntry.ExtendedAttributes, "atime"); - expectedCTime = formatToConvert is TarEntryFormat.Gnu ? default : GetDateTimeOffsetFromTimestampString(originalPaxEntry.ExtendedAttributes, "ctime"); + + expectedATime = null; + if (originalPaxEntry.ExtendedAttributes.ContainsKey("atime")) + { + expectedATime = GetDateTimeOffsetFromTimestampString(originalPaxEntry.ExtendedAttributes, "atime"); + } + + expectedCTime = null; + if (originalPaxEntry.ExtendedAttributes.ContainsKey("ctime")) + { + expectedCTime = GetDateTimeOffsetFromTimestampString(originalPaxEntry.ExtendedAttributes, "ctime"); + } } - else + else if (originalEntry.Format is TarEntryFormat.Gnu) { GnuTarEntry originalGnuEntry = originalEntry as GnuTarEntry; expectedATime = originalGnuEntry.AccessTime; + // default means: no timestamp. + if (expectedATime == default(DateTimeOffset)) + { + expectedATime = null; + } expectedCTime = originalGnuEntry.ChangeTime; + if (expectedCTime == default(DateTimeOffset)) + { + expectedCTime = null; + } + } + else + { + // Format has no timestamps. + expectedATime = null; + expectedCTime = null; } - } protected TarEntry InvokeTarEntryConversionConstructor(TarEntryFormat targetFormat, TarEntry other) @@ -220,5 +180,11 @@ protected TarEntry InvokeTarEntryConversionConstructor(TarEntryFormat targetForm TarEntryFormat.Gnu => new GnuTarEntry(other), _ => throw new InvalidDataException($"Unexpected format: {targetFormat}") }; + + [Fact] + public void Constructor_ConversionFromGnu_ATimeCTime() => TestConstructionConversion(TarEntryType.RegularFile, TarEntryFormat.Gnu, FormatUnderTest, setATimeCTime: true); + + [Fact] + public void Constructor_ConversionFromPax_ATimeCTime() => TestConstructionConversion(TarEntryType.RegularFile, TarEntryFormat.Pax, FormatUnderTest, setATimeCTime: true); } } diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/UstarTarEntry.Conversion.Tests.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/UstarTarEntry.Conversion.Tests.cs index 4cec52322be..ef366d415f6 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/UstarTarEntry.Conversion.Tests.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/UstarTarEntry.Conversion.Tests.cs @@ -13,6 +13,8 @@ namespace System.Formats.Tar.Tests { public class UstarTarEntry_Conversion_Tests : TarTestsConversionBase { + protected override TarEntryFormat FormatUnderTest => TarEntryFormat.Ustar; + [Fact] public void Constructor_ConversionFromV7_RegularFile() => TestConstructionConversion(TarEntryType.RegularFile, TarEntryFormat.V7, TarEntryFormat.Ustar); diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/V7TarEntry.Conversion.Tests.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/V7TarEntry.Conversion.Tests.cs index 8856ec4d548..e6293999af0 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/V7TarEntry.Conversion.Tests.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarEntry/V7TarEntry.Conversion.Tests.cs @@ -13,6 +13,8 @@ namespace System.Formats.Tar.Tests { public class V7TarEntry_Conversion_Tests : TarTestsConversionBase { + protected override TarEntryFormat FormatUnderTest => TarEntryFormat.V7; + [Fact] public void Constructor_Conversion_UnsupportedEntryTypes_Ustar() { diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.Tests.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.Tests.cs index f95a22335a1..38a94515afa 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.Tests.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.Tests.cs @@ -113,7 +113,7 @@ public void PaxExtendedAttribute_Roundtrips(string key, string value) using (var reader = new TarReader(stream)) { PaxTarEntry entry = Assert.IsType(reader.GetNextEntry()); - Assert.Equal(5, entry.ExtendedAttributes.Count); + Assert.Equal(3, entry.ExtendedAttributes.Count); Assert.Contains(KeyValuePair.Create(key, value), entry.ExtendedAttributes); Assert.Null(reader.GetNextEntry()); } diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarTestsBase.Pax.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarTestsBase.Pax.cs index adfbea8fdaa..9946c6ce437 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarTestsBase.Pax.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarTestsBase.Pax.cs @@ -87,8 +87,14 @@ protected void VerifyFifo(PaxTarEntry fifo) private DateTimeOffset GetDateTimeOffsetFromSecondsSinceEpoch(decimal secondsSinceUnixEpoch) => new DateTimeOffset((long)(secondsSinceUnixEpoch * TimeSpan.TicksPerSecond) + DateTime.UnixEpoch.Ticks, TimeSpan.Zero); - private decimal GetSecondsSinceEpochFromDateTimeOffset(DateTimeOffset value) => - ((decimal)(value.UtcDateTime - DateTime.UnixEpoch).Ticks) / TimeSpan.TicksPerSecond; + protected DateTimeOffset? TryGetDateTimeOffsetFromTimestampString(IReadOnlyDictionary ea, string fieldName) + { + if (!ea.ContainsKey(fieldName)) + { + return default; + } + return GetDateTimeOffsetFromTimestampString(ea[fieldName]); + } protected DateTimeOffset GetDateTimeOffsetFromTimestampString(IReadOnlyDictionary ea, string fieldName) { @@ -102,12 +108,6 @@ protected DateTimeOffset GetDateTimeOffsetFromTimestampString(string strNumber) return GetDateTimeOffsetFromSecondsSinceEpoch(secondsSinceEpoch); } - protected string GetTimestampStringFromDateTimeOffset(DateTimeOffset timestamp) - { - decimal secondsSinceEpoch = GetSecondsSinceEpochFromDateTimeOffset(timestamp); - return secondsSinceEpoch.ToString("G", CultureInfo.InvariantCulture); - } - protected void VerifyExtendedAttributeTimestamp(PaxTarEntry paxEntry, string fieldName, DateTimeOffset minimumTime) { DateTimeOffset converted = GetDateTimeOffsetFromTimestampString(paxEntry.ExtendedAttributes, fieldName); @@ -117,11 +117,9 @@ protected void VerifyExtendedAttributeTimestamp(PaxTarEntry paxEntry, string fie protected void VerifyExtendedAttributeTimestamps(PaxTarEntry pax) { Assert.NotNull(pax.ExtendedAttributes); - AssertExtensions.GreaterThanOrEqualTo(pax.ExtendedAttributes.Count, 3); // Expect to at least collect mtime, ctime and atime + AssertExtensions.GreaterThanOrEqualTo(pax.ExtendedAttributes.Count, 1); // Expect to at least collect mtime VerifyExtendedAttributeTimestamp(pax, PaxEaMTime, MinimumTime); - VerifyExtendedAttributeTimestamp(pax, PaxEaATime, MinimumTime); - VerifyExtendedAttributeTimestamp(pax, PaxEaCTime, MinimumTime); } public static IEnumerable GetPaxExtendedAttributesRoundtripTestData() diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs index e8df0e4a9a3..b68493996a0 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Runtime.CompilerServices; @@ -222,6 +223,15 @@ protected TarTestsBase() 8); } + private static decimal GetSecondsSinceEpochFromDateTimeOffset(DateTimeOffset value) => + ((decimal)(value.UtcDateTime - DateTime.UnixEpoch).Ticks) / TimeSpan.TicksPerSecond; + + protected static string GetTimestampStringFromDateTimeOffset(DateTimeOffset timestamp) + { + decimal secondsSinceEpoch = GetSecondsSinceEpochFromDateTimeOffset(timestamp); + return secondsSinceEpoch.ToString("G", CultureInfo.InvariantCulture); + } + protected static string GetTestCaseUnarchivedFolderPath(string testCaseName) => Path.Join(Directory.GetCurrentDirectory(), "unarchived", testCaseName); @@ -493,16 +503,57 @@ protected static TarEntryType GetTarEntryTypeForTarEntryFormat(TarEntryType entr return entryType; } - protected static TarEntry InvokeTarEntryCreationConstructor(TarEntryFormat targetFormat, TarEntryType entryType, string entryName) - => targetFormat switch + protected TarEntry InvokeTarEntryCreationConstructor(TarEntryFormat targetFormat, TarEntryType entryType, string entryName, bool setATimeCTime = false) + { + TarEntry entry = (targetFormat, setATimeCTime) switch { - TarEntryFormat.V7 => new V7TarEntry(entryType, entryName), - TarEntryFormat.Ustar => new UstarTarEntry(entryType, entryName), - TarEntryFormat.Pax => new PaxTarEntry(entryType, entryName), - TarEntryFormat.Gnu => new GnuTarEntry(entryType, entryName), + (TarEntryFormat.V7, _) => new V7TarEntry(entryType, entryName), + (TarEntryFormat.Ustar, _) => new UstarTarEntry(entryType, entryName), + (TarEntryFormat.Pax, true) => new PaxTarEntry(entryType, entryName, CreateATimeCTimeExtendedAttributes()), + (TarEntryFormat.Pax, false) => new PaxTarEntry(entryType, entryName), + (TarEntryFormat.Gnu, _) => new GnuTarEntry(entryType, entryName), _ => throw new InvalidDataException($"Unexpected format: {targetFormat}") }; + if (entry is GnuTarEntry gnuTarEntry) + { + if (setATimeCTime) + { + gnuTarEntry.AccessTime = TestAccessTime; + gnuTarEntry.ChangeTime = TestChangeTime; + } + else + { + Assert.Equal(default, gnuTarEntry.AccessTime); + Assert.Equal(default, gnuTarEntry.ChangeTime); + } + } + else if (entry is PaxTarEntry paxTarEntry) + { + if (setATimeCTime) + { + Assert.Contains("ctime", paxTarEntry.ExtendedAttributes); + Assert.Contains("atime", paxTarEntry.ExtendedAttributes); + } + else + { + Assert.DoesNotContain("ctime", paxTarEntry.ExtendedAttributes); + Assert.DoesNotContain("atime", paxTarEntry.ExtendedAttributes); + } + } + + return entry; + } + + private Dictionary CreateATimeCTimeExtendedAttributes() + { + return new Dictionary() + { + { "atime", GetTimestampStringFromDateTimeOffset(TestAccessTime) }, + { "ctime", GetTimestampStringFromDateTimeOffset(TestChangeTime) }, + }; + } + public static IEnumerable GetTestTarFormats() { foreach (TestTarFormat testFormat in Enum.GetValues()) @@ -837,7 +888,7 @@ internal enum NameCapabilities Unlimited } - internal static void WriteTarArchiveWithOneEntry(Stream s, TarEntryFormat entryFormat, TarEntryType entryType) + internal void WriteTarArchiveWithOneEntry(Stream s, TarEntryFormat entryFormat, TarEntryType entryType) { using TarWriter writer = new(s, leaveOpen: true); diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs index 006aace6837..840cc8946ea 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs @@ -182,12 +182,10 @@ public void WritePaxAttributes_CustomAttribute() Assert.NotNull(regularFile.ExtendedAttributes); // path, mtime, atime and ctime are always collected by default - AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 5); + AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 3); Assert.Contains(PaxEaName, regularFile.ExtendedAttributes); Assert.Contains(PaxEaMTime, regularFile.ExtendedAttributes); - Assert.Contains(PaxEaATime, regularFile.ExtendedAttributes); - Assert.Contains(PaxEaCTime, regularFile.ExtendedAttributes); Assert.Contains(expectedKey, regularFile.ExtendedAttributes); Assert.Equal(expectedValue, regularFile.ExtendedAttributes[expectedKey]); @@ -210,10 +208,8 @@ public void WritePaxAttributes_Timestamps_AutomaticallyAdded() { PaxTarEntry regularFile = reader.GetNextEntry() as PaxTarEntry; - AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 4); + AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 2); VerifyExtendedAttributeTimestamp(regularFile, PaxEaMTime, minimumTime); - VerifyExtendedAttributeTimestamp(regularFile, PaxEaATime, minimumTime); - VerifyExtendedAttributeTimestamp(regularFile, PaxEaCTime, minimumTime); } } @@ -269,13 +265,11 @@ public void WritePaxAttributes_LongGroupName_LongUserName() Assert.NotNull(regularFile.ExtendedAttributes); - // path, mtime, atime and ctime are always collected by default - AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 6); + // path, mtime are always collected by default + AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 4); Assert.Contains(PaxEaName, regularFile.ExtendedAttributes); Assert.Contains(PaxEaMTime, regularFile.ExtendedAttributes); - Assert.Contains(PaxEaATime, regularFile.ExtendedAttributes); - Assert.Contains(PaxEaCTime, regularFile.ExtendedAttributes); Assert.Contains(PaxEaUName, regularFile.ExtendedAttributes); Assert.Equal(userName, regularFile.ExtendedAttributes[PaxEaUName]); @@ -304,7 +298,7 @@ public void WritePaxAttributes_Name_AutomaticallyAdded() { PaxTarEntry regularFile = reader.GetNextEntry() as PaxTarEntry; - AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 4); + AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 2); Assert.Contains(PaxEaName, regularFile.ExtendedAttributes); } } @@ -332,7 +326,7 @@ public void WritePaxAttributes_LongLinkName_AutomaticallyAdded() { PaxTarEntry symlink = reader.GetNextEntry() as PaxTarEntry; - AssertExtensions.GreaterThanOrEqualTo(symlink.ExtendedAttributes.Count, 5); + AssertExtensions.GreaterThanOrEqualTo(symlink.ExtendedAttributes.Count, 3); Assert.Contains(PaxEaName, symlink.ExtendedAttributes); Assert.Equal("symlink", symlink.ExtendedAttributes[PaxEaName]); @@ -341,7 +335,7 @@ public void WritePaxAttributes_LongLinkName_AutomaticallyAdded() PaxTarEntry hardlink = reader.GetNextEntry() as PaxTarEntry; - AssertExtensions.GreaterThanOrEqualTo(hardlink.ExtendedAttributes.Count, 5); + AssertExtensions.GreaterThanOrEqualTo(hardlink.ExtendedAttributes.Count, 3); Assert.Contains(PaxEaName, hardlink.ExtendedAttributes); Assert.Equal("hardlink", hardlink.ExtendedAttributes[PaxEaName]); diff --git a/src/runtime/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs b/src/runtime/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs index b20d1c78e1a..160444743a0 100644 --- a/src/runtime/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs +++ b/src/runtime/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs @@ -198,12 +198,10 @@ public async Task WritePaxAttributes_CustomAttribute_Async() Assert.NotNull(regularFile.ExtendedAttributes); // path, mtime, atime and ctime are always collected by default - AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 5); + AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 3); Assert.Contains(PaxEaName, regularFile.ExtendedAttributes); Assert.Contains(PaxEaMTime, regularFile.ExtendedAttributes); - Assert.Contains(PaxEaATime, regularFile.ExtendedAttributes); - Assert.Contains(PaxEaCTime, regularFile.ExtendedAttributes); Assert.Contains(expectedKey, regularFile.ExtendedAttributes); Assert.Equal(expectedValue, regularFile.ExtendedAttributes[expectedKey]); @@ -228,10 +226,8 @@ public async Task WritePaxAttributes_Timestamps_AutomaticallyAdded_Async() { PaxTarEntry regularFile = await reader.GetNextEntryAsync() as PaxTarEntry; - AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 4); + AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 2); VerifyExtendedAttributeTimestamp(regularFile, PaxEaMTime, minimumTime); - VerifyExtendedAttributeTimestamp(regularFile, PaxEaATime, minimumTime); - VerifyExtendedAttributeTimestamp(regularFile, PaxEaCTime, minimumTime); } } } @@ -291,12 +287,10 @@ public async Task WritePaxAttributes_LongGroupName_LongUserName_Async() Assert.NotNull(regularFile.ExtendedAttributes); // path, mtime, atime and ctime are always collected by default - AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 6); + AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 4); Assert.Contains(PaxEaName, regularFile.ExtendedAttributes); Assert.Contains(PaxEaMTime, regularFile.ExtendedAttributes); - Assert.Contains(PaxEaATime, regularFile.ExtendedAttributes); - Assert.Contains(PaxEaCTime, regularFile.ExtendedAttributes); Assert.Contains(PaxEaUName, regularFile.ExtendedAttributes); Assert.Equal(userName, regularFile.ExtendedAttributes[PaxEaUName]); @@ -325,7 +319,7 @@ public async Task WritePaxAttributes_Name_AutomaticallyAdded_Async() { PaxTarEntry regularFile = await reader.GetNextEntryAsync() as PaxTarEntry; - AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 4); + AssertExtensions.GreaterThanOrEqualTo(regularFile.ExtendedAttributes.Count, 2); Assert.Contains(PaxEaName, regularFile.ExtendedAttributes); } } @@ -353,7 +347,7 @@ public async Task WritePaxAttributes_LongLinkName_AutomaticallyAdded_Async() { PaxTarEntry symlink = await reader.GetNextEntryAsync() as PaxTarEntry; - AssertExtensions.GreaterThanOrEqualTo(symlink.ExtendedAttributes.Count, 5); + AssertExtensions.GreaterThanOrEqualTo(symlink.ExtendedAttributes.Count, 3); Assert.Contains(PaxEaName, symlink.ExtendedAttributes); Assert.Equal("symlink", symlink.ExtendedAttributes[PaxEaName]); @@ -362,7 +356,7 @@ public async Task WritePaxAttributes_LongLinkName_AutomaticallyAdded_Async() PaxTarEntry hardlink = await reader.GetNextEntryAsync() as PaxTarEntry; - AssertExtensions.GreaterThanOrEqualTo(hardlink.ExtendedAttributes.Count, 5); + AssertExtensions.GreaterThanOrEqualTo(hardlink.ExtendedAttributes.Count, 3); Assert.Contains(PaxEaName, hardlink.ExtendedAttributes); Assert.Equal("hardlink", hardlink.ExtendedAttributes[PaxEaName]); diff --git a/src/runtime/src/libraries/System.Private.CoreLib/gen/NativeRuntimeEventSourceGenerator.cs b/src/runtime/src/libraries/System.Private.CoreLib/gen/NativeRuntimeEventSourceGenerator.cs new file mode 100644 index 00000000000..5d8c66a6008 --- /dev/null +++ b/src/runtime/src/libraries/System.Private.CoreLib/gen/NativeRuntimeEventSourceGenerator.cs @@ -0,0 +1,424 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; + +namespace Generators +{ + [Generator] + public sealed class NativeRuntimeEventSourceGenerator : IIncrementalGenerator + { + private static readonly XNamespace EventNs = "http://schemas.microsoft.com/win/2004/08/events"; + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + IncrementalValuesProvider manifestFiles = context.AdditionalTextsProvider.Where(f => f.Path.EndsWith(".man", StringComparison.OrdinalIgnoreCase)); + IncrementalValuesProvider inclusionFiles = context.AdditionalTextsProvider.Where(f => f.Path.EndsWith(".lst", StringComparison.OrdinalIgnoreCase)); + + IncrementalValuesProvider<(AdditionalText Left, System.Collections.Immutable.ImmutableArray Right)> combined = manifestFiles.Combine(inclusionFiles.Collect()); + + context.RegisterSourceOutput(combined, (spc, tuple) => + { + AdditionalText manifestFile = tuple.Left; + System.Collections.Immutable.ImmutableArray inclusionFiles = tuple.Right; + string manifestText = manifestFile.GetText(spc.CancellationToken)?.ToString(); + if (string.IsNullOrEmpty(manifestText)) + { + return; + } + + var manifest = XDocument.Parse(manifestText); + + string inclusionText = inclusionFiles.FirstOrDefault()?.GetText(spc.CancellationToken)?.ToString(); + + Dictionary> inclusionList = ParseInclusionListFromString(inclusionText); + + foreach (KeyValuePair kvp in manifestsToGenerate) + { + string providerName = kvp.Key; + string className = providerNameToClassNameMap[providerName]; + XElement? providerNode = manifest + .Descendants(EventNs + "provider") + .FirstOrDefault(e => (string)e.Attribute("name") == providerName); + + if (providerNode is null) + { + continue; + } + + string source = GenerateEventSourceClass(providerNode, className, inclusionList); + spc.AddSource($"{className}.g.cs", SourceText.From(source, System.Text.Encoding.UTF8)); + } + }); + } + + private static Dictionary> ParseInclusionListFromString(string? inclusionText) + { + Dictionary> inclusionList = []; + if (string.IsNullOrEmpty(inclusionText)) + { + return inclusionList; + } + + using var reader = new StringReader(inclusionText); + string line; + while ((line = reader.ReadLine()) != null) + { + string trimmed = line.Trim(); + if (string.IsNullOrEmpty(trimmed) || trimmed.StartsWith("#")) + { + continue; + } + + string[] tokens = trimmed.Split(':'); + if (tokens.Length == 0) + { + continue; + } + + if (tokens.Length > 2) + { + continue; + } + + string providerName, eventName; + if (tokens.Length == 2) + { + providerName = tokens[0]; + eventName = tokens[1]; + } + else + { + providerName = "*"; + eventName = tokens[0]; + } + if (!inclusionList.TryGetValue(providerName, out HashSet? value)) + { + value = []; + inclusionList[providerName] = value; + } + + value.Add(eventName); + } + return inclusionList; + } + + private static bool IncludeEvent(Dictionary> inclusionList, string providerName, string eventName) + { + if (inclusionList == null || inclusionList.Count == 0) + { + return true; + } + + if (inclusionList.TryGetValue(providerName, out HashSet? events) && events.Contains(eventName)) + { + return true; + } + + if (inclusionList.TryGetValue("*", out HashSet? wildcardEvents) && wildcardEvents.Contains(eventName)) + { + return true; + } + + return false; + } + + private static string GenerateEventSourceClass(XElement providerNode, string className, Dictionary> inclusionList) + { + var sw = new StringWriter(); + + sw.WriteLine($$""" + // Licensed to the .NET Foundation under one or more agreements. + // The .NET Foundation licenses this file to you under the MIT license. + // + + using System; + + namespace System.Diagnostics.Tracing + { + internal sealed partial class {{className}} : EventSource + { + """); + + GenerateKeywordsClass(providerNode, sw, inclusionList); + GenerateEventMethods(providerNode, sw, inclusionList); + + sw.WriteLine(""" + } + } + """); + return sw.ToString(); + } + + private static void GenerateKeywordsClass(XElement providerNode, StringWriter writer, Dictionary> inclusionList) + { + string? providerName = providerNode.Attribute("name")?.Value; + + if (providerName is null) + { + return; + } + + XElement eventsNode = providerNode.Element(EventNs + "events"); + if (eventsNode is null) + { + return; + } + + IEnumerable eventNodes = eventsNode.Elements(EventNs + "event"); + var usedKeywords = new HashSet(); + foreach (XElement? eventNode in eventNodes) + { + string? eventName = eventNode.Attribute("symbol")?.Value; + + if (eventName is null + || !IncludeEvent(inclusionList, providerName, eventName)) + { + continue; + } + + string? keywords = eventNode.Attribute("keywords")?.Value; + if (!string.IsNullOrEmpty(keywords)) + { + foreach (string? kw in keywords.Split([' '], StringSplitOptions.RemoveEmptyEntries)) + { + usedKeywords.Add(kw); + } + } + } + XElement? keywordsNode = providerNode.Element(EventNs + "keywords"); + if (keywordsNode is null) + { + return; + } + + writer.WriteLine(""" + public static class Keywords + { + """); + + foreach (XElement keywordNode in keywordsNode.Elements(EventNs + "keyword")) + { + string? name = keywordNode.Attribute("name")?.Value; + string? mask = keywordNode.Attribute("mask")?.Value; + if (name is not null && mask is not null && usedKeywords.Contains(name)) + { + writer.WriteLine($" public const EventKeywords {name} = (EventKeywords){mask};"); + } + } + + writer.WriteLine(""" + } + + """); + } + + private static void GenerateEventMethods(XElement providerNode, StringWriter writer, Dictionary> inclusionList) + { + string? providerName = providerNode.Attribute("name")?.Value; + + if (providerName is null) + { + return; + } + + XElement eventsNode = providerNode.Element(EventNs + "events"); + if (eventsNode == null) + { + return; + } + + var eventNodes = eventsNode.Elements(EventNs + "event").ToList(); + XElement templatesNode = providerNode.Element(EventNs + "templates"); + var templateDict = new Dictionary(); + if (templatesNode != null) + { + foreach (XElement? template in templatesNode.Elements(EventNs + "template")) + { + string? name = template.Attribute("tid")?.Value; + if (!string.IsNullOrEmpty(name)) + { + templateDict[name] = template; + } + } + } + + // Build a dictionary of eventID -> latest version + Dictionary latestEventVersions = []; + foreach (XElement? eventNode in eventNodes) + { + string? eventName = eventNode.Attribute("symbol")?.Value; + if (eventName is null + || !IncludeEvent(inclusionList, providerName, eventName)) + { + continue; + } + + string? eventId = eventNode.Attribute("value")?.Value; + string? version = eventNode.Attribute("version")?.Value; + if (eventId is not null && version is not null) + { + if (!latestEventVersions.TryGetValue(eventId, out string? existingVersion) || string.CompareOrdinal(version, existingVersion) > 0) + { + latestEventVersions[eventId] = version; + } + } + } + + foreach (XElement? eventNode in eventNodes) + { + string? eventName = eventNode.Attribute("symbol")?.Value; + if (eventName is null + || !IncludeEvent(inclusionList, providerName, eventName)) + { + continue; + } + + if (IsEventManuallyHandled(eventName)) + { + continue; + } + + string? eventId = eventNode.Attribute("value")?.Value; + string? version = eventNode.Attribute("version")?.Value; + // Only emit the event if it is the latest version for this eventId + if (eventId is null || version is null || latestEventVersions[eventId] != version) + { + continue; + } + + string? level = eventNode.Attribute("level")?.Value; + IEnumerable? keywords = eventNode + .Attribute("keywords") + ?.Value + .ToString() + .Split([' '], StringSplitOptions.RemoveEmptyEntries) + .Select(k => $"Keywords.{k}"); + + writer.Write($" [Event({eventId}, Version = {version}, Level = EventLevel.{level?.Replace("win:", "")}"); + + if (keywords?.Any() == true) + { + writer.Write($", Keywords = {string.Join(" | ", keywords)}"); + } + + writer.WriteLine(")]"); + + // Write the method signature + writer.Write($" private void {eventName}("); + + string? templateValue = eventNode.Attribute("template")?.Value; + + if (!string.IsNullOrEmpty(templateValue) + && templateDict.TryGetValue(templateValue, out XElement? template)) + { + IEnumerable dataNodes = template.Elements(EventNs + "data").ToArray(); + var paramList = new List(); + + // Calculate the number of arguments to emit. + // COMPAT: Cut the parameter list at any binary or ansi string arguments, + // or if the count attribute is set on any of the parameters. + int numArgumentsToEmit = 0; + foreach (XElement data in dataNodes) + { + string? paramType = data.Attribute("inType")?.Value.ToString(); + + if (paramType is "win:Binary" or "win:AnsiString") + { + break; + } + + if (!string.IsNullOrEmpty(data.Attribute("count")?.Value)) + { + break; + } + + numArgumentsToEmit++; + } + + foreach (XElement data in dataNodes) + { + if (numArgumentsToEmit-- <= 0) + { + break; + } + + string? paramType = data.Attribute("inType")?.Value; + string? paramName = data.Attribute("name")?.Value; + if (paramType is not null && paramName is not null + && manifestTypeToCSharpTypeMap.TryGetValue(paramType, out string? csType)) + { + paramList.Add($"{csType} {paramName}"); + } + else if (paramType is not null && paramName is not null) + { + paramList.Add($"object {paramName}"); + } + } + writer.Write(string.Join(", ", paramList)); + } + + writer.WriteLine(""" + ) + { + // To have this event be emitted from managed side, refer to NativeRuntimeEventSource.cs + throw new NotImplementedException(); + } + + """); + } + } + + private static bool IsEventManuallyHandled(string eventName) + { + foreach (string handledEvent in manuallyHandledEventSymbols) + { + if (eventName.StartsWith(handledEvent, StringComparison.Ordinal)) + { + return true; + } + } + return false; + } + + private static readonly Dictionary manifestsToGenerate = new() + { + { "Microsoft-Windows-DotNETRuntime", "NativeRuntimeEventSource.Generated.cs" } + }; + + private static readonly Dictionary providerNameToClassNameMap = new() + { + { "Microsoft-Windows-DotNETRuntime", "NativeRuntimeEventSource" } + }; + + private static readonly Dictionary manifestTypeToCSharpTypeMap = new() + { + { "win:UInt8", "byte" }, + { "win:UInt16", "ushort" }, + { "win:UInt32", "uint" }, + { "win:UInt64", "ulong" }, + { "win:Int32", "int" }, + { "win:Int64", "long" }, + { "win:Pointer", "IntPtr" }, + { "win:UnicodeString", "string" }, + { "win:Binary", "byte[]" }, + { "win:Double", "double" }, + { "win:Boolean", "bool" }, + { "win:GUID", "Guid" }, + }; + + private static readonly List manuallyHandledEventSymbols = + [ + // Some threading events are defined manually in NativeRuntimeEventSource.Threading.cs + "ThreadPool", + "Contention", + "WaitHandle" + ]; + } +} diff --git a/src/runtime/src/libraries/System.Private.CoreLib/gen/System.Private.CoreLib.Generators.csproj b/src/runtime/src/libraries/System.Private.CoreLib/gen/System.Private.CoreLib.Generators.csproj index 9abeb30c867..bb2ffa89ff7 100644 --- a/src/runtime/src/libraries/System.Private.CoreLib/gen/System.Private.CoreLib.Generators.csproj +++ b/src/runtime/src/libraries/System.Private.CoreLib/gen/System.Private.CoreLib.Generators.csproj @@ -2,6 +2,7 @@ netstandard2.0 $(NoWarn);CS3001 + true $(DefineConstants);STABILIZE_PACKAGE_VERSION @@ -13,6 +14,7 @@ + @@ -20,7 +22,7 @@ - + diff --git a/src/runtime/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/runtime/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index cae62dc6eb8..8f60c60d08f 100644 --- a/src/runtime/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/runtime/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -6082,6 +6082,11 @@ public bool AppendFormatted(T value) return AppendCustomFormatter(value, format: null); } + if (value is null) + { + return true; + } + // Check first for IFormattable, even though we'll prefer to use ISpanFormattable, as the latter // derives from the former. For value types, it won't matter as the type checks devolve into // JIT-time constants. For reference types, they're more likely to implement IFormattable @@ -6120,7 +6125,7 @@ public bool AppendFormatted(T value) } else { - s = value?.ToString(); + s = value.ToString(); } return s is null || AppendLiteral(s); @@ -6138,6 +6143,11 @@ public bool AppendFormatted(T value, string? format) return AppendCustomFormatter(value, format); } + if (value is null) + { + return true; + } + // Check first for IFormattable, even though we'll prefer to use ISpanFormattable, as the latter // derives from the former. For value types, it won't matter as the type checks devolve into // JIT-time constants. For reference types, they're more likely to implement IFormattable @@ -6176,7 +6186,7 @@ public bool AppendFormatted(T value, string? format) } else { - s = value?.ToString(); + s = value.ToString(); } return s is null || AppendLiteral(s); diff --git a/src/runtime/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/DefaultInterpolatedStringHandler.cs b/src/runtime/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/DefaultInterpolatedStringHandler.cs index fda5d9b435b..0fa80dc2600 100644 --- a/src/runtime/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/DefaultInterpolatedStringHandler.cs +++ b/src/runtime/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/DefaultInterpolatedStringHandler.cs @@ -247,6 +247,11 @@ public void AppendFormatted(T value) return; } + if (value is null) + { + return; + } + // Check first for IFormattable, even though we'll prefer to use ISpanFormattable, as the latter // requires the former. For value types, it won't matter as the type checks devolve into // JIT-time constants. For reference types, they're more likely to implement IFormattable @@ -287,7 +292,7 @@ public void AppendFormatted(T value) } else { - s = value?.ToString(); + s = value.ToString(); } if (s is not null) @@ -309,6 +314,11 @@ public void AppendFormatted(T value, string? format) return; } + if (value is null) + { + return; + } + // Check first for IFormattable, even though we'll prefer to use ISpanFormattable, as the latter // requires the former. For value types, it won't matter as the type checks devolve into // JIT-time constants. For reference types, they're more likely to implement IFormattable @@ -349,7 +359,7 @@ public void AppendFormatted(T value, string? format) } else { - s = value?.ToString(); + s = value.ToString(); } if (s is not null) diff --git a/src/runtime/src/libraries/System.Private.CoreLib/src/System/Text/StringBuilder.cs b/src/runtime/src/libraries/System.Private.CoreLib/src/System/Text/StringBuilder.cs index 11b76d7b195..a159895c0ed 100644 --- a/src/runtime/src/libraries/System.Private.CoreLib/src/System/Text/StringBuilder.cs +++ b/src/runtime/src/libraries/System.Private.CoreLib/src/System/Text/StringBuilder.cs @@ -2869,8 +2869,15 @@ public void AppendFormatted(T value) { // If there's a custom formatter, always use it. AppendCustomFormatter(value, format: null); + return; } - else if (value is IFormattable) + + if (value is null) + { + return; + } + + if (value is IFormattable) { // Check first for IFormattable, even though we'll prefer to use ISpanFormattable, as the latter // requires the former. For value types, it won't matter as the type checks devolve into @@ -2917,7 +2924,7 @@ public void AppendFormatted(T value) _stringBuilder.Append(((IFormattable)value).ToString(format: null, _provider)); // constrained call avoiding boxing for value types } } - else if (value is not null) + else { _stringBuilder.Append(value.ToString()); } @@ -2933,8 +2940,15 @@ public void AppendFormatted(T value, string? format) { // If there's a custom formatter, always use it. AppendCustomFormatter(value, format); + return; } - else if (value is IFormattable) + + if (value is null) + { + return; + } + + if (value is IFormattable) { // Check first for IFormattable, even though we'll prefer to use ISpanFormattable, as the latter // requires the former. For value types, it won't matter as the type checks devolve into @@ -2981,7 +2995,7 @@ public void AppendFormatted(T value, string? format) _stringBuilder.Append(((IFormattable)value).ToString(format, _provider)); // constrained call avoiding boxing for value types } } - else if (value is not null) + else { _stringBuilder.Append(value.ToString()); } diff --git a/src/runtime/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs b/src/runtime/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs index e1aa47dc57b..b36be6cab83 100644 --- a/src/runtime/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs +++ b/src/runtime/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs @@ -432,6 +432,11 @@ public bool AppendFormatted(T value) return AppendCustomFormatter(value, format: null); } + if (value is null) + { + return true; + } + // Special-case enums to avoid boxing them. if (typeof(T).IsEnum) { @@ -467,7 +472,7 @@ public bool AppendFormatted(T value) else { // Fall back to a normal ToString and append that. - s = value?.ToString(); + s = value.ToString(); } return AppendFormatted(s.AsSpan()); @@ -485,6 +490,11 @@ public bool AppendFormatted(T value, string? format) return AppendCustomFormatter(value, format); } + if (value is null) + { + return true; + } + // Special-case enums to avoid boxing them. if (typeof(T).IsEnum) { @@ -520,7 +530,7 @@ public bool AppendFormatted(T value, string? format) else { // Fall back to a normal ToString and append that. - s = value?.ToString(); + s = value.ToString(); } return AppendFormatted(s.AsSpan()); diff --git a/src/runtime/src/libraries/System.Reflection.TypeExtensions/ref/System.Reflection.TypeExtensions.cs b/src/runtime/src/libraries/System.Reflection.TypeExtensions/ref/System.Reflection.TypeExtensions.cs index c417b43e94a..8395088bcfb 100644 --- a/src/runtime/src/libraries/System.Reflection.TypeExtensions/ref/System.Reflection.TypeExtensions.cs +++ b/src/runtime/src/libraries/System.Reflection.TypeExtensions/ref/System.Reflection.TypeExtensions.cs @@ -104,11 +104,11 @@ public static partial class TypeExtensions [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static System.Reflection.MemberInfo[] GetMember([System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicEvents | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicMethods | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicNestedTypes | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] this System.Type type, string name) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] - public static System.Reflection.MemberInfo[] GetMember([System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.All)] this System.Type type, string name, System.Reflection.BindingFlags bindingAttr) { throw null; } + public static System.Reflection.MemberInfo[] GetMember([System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicEvents | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicMethods | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicNestedTypes | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicProperties | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicEvents | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicMethods | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicNestedTypes | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] this System.Type type, string name, System.Reflection.BindingFlags bindingAttr) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static System.Reflection.MemberInfo[] GetMembers([System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicEvents | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicMethods | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicNestedTypes | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] this System.Type type) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] - public static System.Reflection.MemberInfo[] GetMembers([System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.All)] this System.Type type, System.Reflection.BindingFlags bindingAttr) { throw null; } + public static System.Reflection.MemberInfo[] GetMembers([System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicEvents | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicMethods | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicNestedTypes | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicProperties | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicEvents | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicFields | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicMethods | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicNestedTypes | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties)] this System.Type type, System.Reflection.BindingFlags bindingAttr) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static System.Reflection.MethodInfo? GetMethod([System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicMethods)] this System.Type type, string name) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] diff --git a/src/runtime/src/libraries/System.Reflection.TypeExtensions/src/System/Reflection/TypeExtensions.cs b/src/runtime/src/libraries/System.Reflection.TypeExtensions/src/System/Reflection/TypeExtensions.cs index b74110f5f45..297518fa50b 100644 --- a/src/runtime/src/libraries/System.Reflection.TypeExtensions/src/System/Reflection/TypeExtensions.cs +++ b/src/runtime/src/libraries/System.Reflection.TypeExtensions/src/System/Reflection/TypeExtensions.cs @@ -168,7 +168,13 @@ public static MemberInfo[] GetMember( [EditorBrowsable(EditorBrowsableState.Never)] public static MemberInfo[] GetMember( - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] this Type type, + [DynamicallyAccessedMembers( + DynamicallyAccessedMemberTypes.PublicFields | DynamicallyAccessedMemberTypes.NonPublicFields | + DynamicallyAccessedMemberTypes.PublicMethods | DynamicallyAccessedMemberTypes.NonPublicMethods | + DynamicallyAccessedMemberTypes.PublicEvents | DynamicallyAccessedMemberTypes.NonPublicEvents | + DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.NonPublicProperties | + DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | + DynamicallyAccessedMemberTypes.PublicNestedTypes | DynamicallyAccessedMemberTypes.NonPublicNestedTypes)] this Type type, string name, BindingFlags bindingAttr) { @@ -194,7 +200,13 @@ public static MemberInfo[] GetMembers( [EditorBrowsable(EditorBrowsableState.Never)] public static MemberInfo[] GetMembers( - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] this Type type, + [DynamicallyAccessedMembers( + DynamicallyAccessedMemberTypes.PublicFields | DynamicallyAccessedMemberTypes.NonPublicFields | + DynamicallyAccessedMemberTypes.PublicMethods | DynamicallyAccessedMemberTypes.NonPublicMethods | + DynamicallyAccessedMemberTypes.PublicEvents | DynamicallyAccessedMemberTypes.NonPublicEvents | + DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.NonPublicProperties | + DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | + DynamicallyAccessedMemberTypes.PublicNestedTypes | DynamicallyAccessedMemberTypes.NonPublicNestedTypes)] this Type type, BindingFlags bindingAttr) { ArgumentNullException.ThrowIfNull(type); diff --git a/src/runtime/src/libraries/System.Speech/src/Recognition/RecognizerBase.cs b/src/runtime/src/libraries/System.Speech/src/Recognition/RecognizerBase.cs index 1d8f932513a..a8773c79d46 100644 --- a/src/runtime/src/libraries/System.Speech/src/Recognition/RecognizerBase.cs +++ b/src/runtime/src/libraries/System.Speech/src/Recognition/RecognizerBase.cs @@ -2101,7 +2101,7 @@ private void DispatchEvents(object eventData) lock (_thisObjectLock) { SpeechEvent speechEvent = eventData as SpeechEvent; - if (!_disposed && eventData != null) + if (!_disposed && speechEvent != null) { switch (speechEvent.EventId) { diff --git a/src/runtime/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonProperty.cs b/src/runtime/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonProperty.cs index 84675bc0b4e..ef120f791f0 100644 --- a/src/runtime/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonProperty.cs +++ b/src/runtime/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonProperty.cs @@ -16,7 +16,6 @@ public readonly struct JsonProperty /// The value of this property. /// public JsonElement Value { get; } - private string? _name { get; } internal JsonProperty(JsonElement value) { @@ -25,8 +24,9 @@ internal JsonProperty(JsonElement value) /// /// The name of this property. + /// This allocates a new string instance for each call. /// - public string Name => _name ?? Value.GetPropertyName(); + public string Name => Value.GetPropertyName(); /// /// Compares to the name of this property. @@ -116,15 +116,7 @@ public void WriteTo(Utf8JsonWriter writer) { ArgumentNullException.ThrowIfNull(writer); - if (_name is null) - { - Value.WritePropertyNameTo(writer); - } - else - { - writer.WritePropertyName(_name); - } - + Value.WritePropertyNameTo(writer); Value.WriteTo(writer); } diff --git a/src/runtime/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/DefaultJsonTypeInfoResolver.Helpers.cs b/src/runtime/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/DefaultJsonTypeInfoResolver.Helpers.cs index 6d5c6acf32f..d7aef7399c0 100644 --- a/src/runtime/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/DefaultJsonTypeInfoResolver.Helpers.cs +++ b/src/runtime/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/DefaultJsonTypeInfoResolver.Helpers.cs @@ -455,7 +455,7 @@ internal static void DeterminePropertyAccessors(JsonPropertyInfo jsonPrope if (converter.ConstructorInfo != null && !converter.ConstructorIsParameterized) { // A parameterless constructor has been resolved by the converter - // (e.g. it might be a non-public ctor with JsonConverterAttribute). + // (e.g. it might be a non-public ctor with JsonConstructorAttribute). defaultCtor = converter.ConstructorInfo; } diff --git a/src/runtime/src/libraries/apicompat/ApiCompatBaseline.NetCoreAppLatestStable.xml b/src/runtime/src/libraries/apicompat/ApiCompatBaseline.NetCoreAppLatestStable.xml index 2c06c8388f1..811dd618a5f 100644 --- a/src/runtime/src/libraries/apicompat/ApiCompatBaseline.NetCoreAppLatestStable.xml +++ b/src/runtime/src/libraries/apicompat/ApiCompatBaseline.NetCoreAppLatestStable.xml @@ -673,6 +673,18 @@ net9.0/System.Reflection.Emit.dll net10.0/System.Reflection.Emit.dll + + CP0015 + M:System.Reflection.TypeExtensions.GetMember(System.Type,System.String,System.Reflection.BindingFlags)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] + net9.0/System.Reflection.TypeExtensions.dll + net10.0/System.Reflection.TypeExtensions.dll + + + CP0015 + M:System.Reflection.TypeExtensions.GetMembers(System.Type,System.Reflection.BindingFlags)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] + net9.0/System.Reflection.TypeExtensions.dll + net10.0/System.Reflection.TypeExtensions.dll + CP0015 M:System.Delegate.#ctor(System.Type,System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] diff --git a/src/runtime/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/runtime/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj index 84945b1e106..1ef2c0a4aa2 100644 --- a/src/runtime/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj +++ b/src/runtime/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj @@ -303,33 +303,7 @@ - - - - - src\System\Diagnostics\Eventing\NativeRuntimeEventSource.Generated.cs - - + + - - - - - - <_PythonWarningParameter>-Wall - <_PythonWarningParameter Condition="'$(MSBuildTreatWarningsAsErrors)' == 'true'">$(_PythonWarningParameter) -Werror - <_EventingSourceFileDirectory>%(EventingSourceFile.RootDir)%(EventingSourceFile.Directory) - <_EventingSourceFileDirectory Condition="HasTrailingSlash('$(_EventingSourceFileDirectory)')">$(_EventingSourceFileDirectory.TrimEnd('\')) - - - - - - - - diff --git a/src/runtime/src/mono/browser/runtime/diagnostics/diagnostics-js.ts b/src/runtime/src/mono/browser/runtime/diagnostics/diagnostics-js.ts index b2fdd3cec88..6ae3d3f63b1 100644 --- a/src/runtime/src/mono/browser/runtime/diagnostics/diagnostics-js.ts +++ b/src/runtime/src/mono/browser/runtime/diagnostics/diagnostics-js.ts @@ -140,6 +140,9 @@ export function cleanupClient () { } export function setupJsClient (client:IDiagnosticClient) { + if (nextJsClient.promise_control.isDone) { + throw new Error("multiple clients in parallel are not allowed"); + } nextJsClient.promise_control.resolve(client); } diff --git a/src/runtime/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorRunOptions.cs b/src/runtime/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorRunOptions.cs index 628227edc7c..c2d27fc91d7 100644 --- a/src/runtime/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorRunOptions.cs +++ b/src/runtime/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorRunOptions.cs @@ -29,6 +29,7 @@ public BlazorRunOptions( string BrowserPath = "", string Locale = "en-US", int? ExpectedExitCode = 0, + int? TimeoutSeconds = 10, string CustomBundleDir = "", bool CheckCounter = true, Func? Test = null, @@ -48,6 +49,7 @@ public BlazorRunOptions( Locale, ExpectedExitCode, CustomBundleDir, + TimeoutSeconds, ExecuteAfterLoaded ) { diff --git a/src/runtime/src/mono/wasm/Wasm.Build.Tests/Blazor/EventPipeDiagnosticsTests.cs b/src/runtime/src/mono/wasm/Wasm.Build.Tests/Blazor/EventPipeDiagnosticsTests.cs new file mode 100644 index 00000000000..8d33d4054d1 --- /dev/null +++ b/src/runtime/src/mono/wasm/Wasm.Build.Tests/Blazor/EventPipeDiagnosticsTests.cs @@ -0,0 +1,346 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Diagnostics.Tracing; +using Microsoft.Diagnostics.Tracing.Etlx; +using Microsoft.Playwright; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests.Blazor; + +public class EventPipeDiagnosticsTests : BlazorWasmTestBase +{ + public EventPipeDiagnosticsTests(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + _enablePerTestCleanup = true; + } + + [Theory] + [InlineData(Configuration.Debug, false)] + [InlineData(Configuration.Release, false)] + [InlineData(Configuration.Release, true)] + public async Task BlazorEventPipeTestWithCpuSamples(Configuration config, bool aot) + { + string extraProperties = @" + all,interval=0 + true + true + "; + + ProjectInfo info = CopyTestAsset(config, aot, TestAsset.BlazorBasicTestApp, "blazor_cpu_samples", extraProperties: extraProperties); + + UpdateCounterPage(); + + BuildProject(info, config, new BuildOptions(AssertAppBundle: false)); + + async Task CpuProfileTest(IPage page) + { + await SetupCounterPage(page); + + await page.EvaluateAsync(@" + globalThis.upload = globalThis.uploadTrace(`cpuprofile.nettrace`, globalThis.getDotnetRuntime(0).collectCpuSamples({ durationSeconds: 2.0, skipDownload: true })); + console.log(`CPU samples collected: ${new Date().toISOString()}`); + "); + + await ClickAndCollect(page); + } + + // Run the test using the custom handler + await RunForBuildWithDotnetRun(new BlazorRunOptions( + Configuration: config, + Test: CpuProfileTest, + TimeoutSeconds: 60, + CheckCounter: false, + ServerEnvironment: new Dictionary + { + ["DEVSERVER_UPLOAD_PATH"] = info.LogPath + } + )); + + var methodFound = false; + using (var source = TraceLog.OpenOrConvert(ConvertTrace(info, "cpuprofile.nettrace"))) + { + methodFound = source.CallStacks.Any(stack => stack.CodeAddress.FullMethodName == "BlazorBasicTestApp.Pages.Counter.IncrementCount()"); + if (!methodFound) + { + foreach (var stack in source.CallStacks) + { + _testOutput.WriteLine($"Stack: {stack.CodeAddress.FullMethodName}"); + } + } + } + + Assert.True(methodFound, "The cpuprofile.nettrace should contain stack frames for the 'Counter.IncrementCount' method"); + } + + [Fact] + public async Task BlazorEventPipeTestWithMetrics() + { + string extraProperties = @" + true + true + "; + + ProjectInfo info = CopyTestAsset(Configuration.Release, aot: false, TestAsset.BlazorBasicTestApp, "blazor_metrics", extraProperties: extraProperties); + + UpdateCounterPage(); + BuildProject(info, Configuration.Release, new BuildOptions(AssertAppBundle: false)); + + async Task CpuProfileTest(IPage page) + { + await SetupCounterPage(page); + + await page.EvaluateAsync(@" + globalThis.upload = globalThis.uploadTrace(`metrics.nettrace`, globalThis.getDotnetRuntime(0).collectMetrics({ durationSeconds: 2.0, skipDownload: true })); + console.log(`Metrics collected: ${new Date().toISOString()}`); + "); + + await ClickAndCollect(page); + } + + // Run the test using the custom handler + await RunForBuildWithDotnetRun(new BlazorRunOptions( + Configuration: Configuration.Release, + Test: CpuProfileTest, + TimeoutSeconds: 60, + CheckCounter: false, + ServerEnvironment: new Dictionary + { + ["DEVSERVER_UPLOAD_PATH"] = info.LogPath + } + )); + + var dictionary = ExtractInstrumentNames(info, "metrics.nettrace"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.assembly.count"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.exceptions"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.gc.collections"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.gc.heap.total_allocated"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.gc.last_collection.heap.fragmentation.size"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.gc.last_collection.heap.size"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.gc.last_collection.memory.committed_size"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.gc.pause.time"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.jit.compilation.time"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.jit.compiled_il.size"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.jit.compiled_methods"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.monitor.lock_contentions"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.process.cpu.count"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.process.memory.working_set"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.thread_pool.queue.length"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.thread_pool.thread.count"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.thread_pool.work_item.count"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("System.Diagnostics.Metrics/instrumentName/dotnet.timer.count"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("Microsoft-DotNETCore-EventPipe/ArchInformation/Unknown"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("Microsoft-DotNETCore-EventPipe/CommandLine//managed BlazorBasicTestApp"), "The metrics.nettrace should contain instrument"); + Assert.True(dictionary.ContainsKey("Microsoft-DotNETCore-EventPipe/OSInformation/Unknown"), "The metrics.nettrace should contain instrument"); + } + + [Fact] + public async Task BlazorEventPipeTestWithHeapDump() + { + string extraProperties = @" + true + true + "; + + ProjectInfo info = CopyTestAsset(Configuration.Release, aot: false, TestAsset.BlazorBasicTestApp, "blazor_gc_dump", extraProperties: extraProperties); + + UpdateCounterPage(); + + // Build the project + BuildProject(info, Configuration.Release, new BuildOptions(AssertAppBundle: false)); + + // Create a custom test handler that will navigate to Counter page, collect CPU samples, + // click the button, and upload the trace + async Task CpuProfileTest(IPage page) + { + await SetupCounterPage(page); + + await page.EvaluateAsync(@" + globalThis.upload = globalThis.uploadTrace(`gcdump.nettrace`, globalThis.getDotnetRuntime(0).collectGcDump({ durationSeconds: 2.0, skipDownload: true })); + console.log(`GC dump collected: ${new Date().toISOString()}`); + "); + + await ClickAndCollect(page); + } + + // Run the test using the custom handler + await RunForBuildWithDotnetRun(new BlazorRunOptions( + Configuration: Configuration.Release, + Test: CpuProfileTest, + TimeoutSeconds: 60, + CheckCounter: false, + ServerEnvironment: new Dictionary + { + ["DEVSERVER_UPLOAD_PATH"] = info.LogPath + } + )); + + var dictionary = ExtractEventNames(info, "gcdump.nettrace"); + Assert.True(dictionary.ContainsKey("Microsoft-Windows-DotNETRuntime/GC/Start"), "The metrics.nettrace should contain GC/Start"); + Assert.True(dictionary.ContainsKey("Microsoft-Windows-DotNETRuntime/GC/Stop"), "The metrics.nettrace should contain GC/Stop"); + Assert.True(dictionary.ContainsKey("Microsoft-Windows-DotNETRuntime/GC/BulkEdge"), "The metrics.nettrace should contain GC/BulkEdge"); + Assert.True(dictionary.ContainsKey("Microsoft-Windows-DotNETRuntime/GC/BulkNode"), "The metrics.nettrace should contain GC/BulkNode"); + Assert.True(dictionary.ContainsKey("Microsoft-Windows-DotNETRuntime/GC/BulkRootEdge"), "The metrics.nettrace should contain GC/BulkRootEdge"); + Assert.True(dictionary.ContainsKey("Microsoft-Windows-DotNETRuntime/Type/BulkType"), "The metrics.nettrace should contain GC/BulkType"); + } + + private string ConvertTrace(ProjectInfo info, string fileName) + { + var traceFilePath = Path.GetFullPath(Path.Combine(info.LogPath, fileName)); + Assert.True(File.Exists(traceFilePath), $"Trace file {traceFilePath} was not created"); + var conversionLog = new StringBuilder(); + using var sw = new StringWriter(conversionLog); + var options = new TraceLogOptions + { + ConversionLog = sw, + }; + var traceConvertedPath = TraceLog.CreateFromEventTraceLogFile(traceFilePath, null, options); + _testOutput.WriteLine(conversionLog.ToString()); + Assert.True(File.Exists(traceConvertedPath), $"Trace file {traceConvertedPath} was not created"); + return traceConvertedPath; + } + + private static Dictionary ExtractEventNames(ProjectInfo info, string fileName) + { + var dictionary = new Dictionary(); + var traceFilePath = Path.GetFullPath(Path.Combine(info.LogPath, fileName)); + using (var source = new EventPipeEventSource(traceFilePath)) + { + source.Clr.All += (data) => + { + var key = $"{data.ProviderName}/{data.EventName}"; + if (dictionary.ContainsKey(key)) + { + dictionary[key] = dictionary[key] + 1; + } + else + { + dictionary[key] = 1; + } + }; + source.Process(); + } + + return dictionary; + } + + private static Dictionary ExtractInstrumentNames(ProjectInfo info, string fileName) + { + var dictionary = new Dictionary(); + var traceFilePath = Path.GetFullPath(Path.Combine(info.LogPath, fileName)); + using (var source = new EventPipeEventSource(traceFilePath)) + { + source.Dynamic.All += (data) => + { + foreach (var arg in data.PayloadNames) + { + var key = $"{data.ProviderName}/{arg}/{data.PayloadByName(arg)}"; + if (dictionary.ContainsKey(key)) + { + dictionary[key] = dictionary[key] + 1; + } + else + { + dictionary[key] = 1; + } + } + }; + source.Process(); + } + + return dictionary; + } + + private void UpdateCounterPage() + { + UpdateFile(Path.Combine("Pages", "Counter.razor"), new Dictionary { + { + @"currentCount++;", + """ + for(int i = 0; i < 100; i++) + { + var sb = new System.Text.StringBuilder(); + sb.Append("Incrementing count: "); + sb.Append(i); + sb.Append(" "); + sb.Append(DateTime.Now.ToString("O")); + if( i % 50 == 0) + { + Console.WriteLine(sb.ToString()); + } + } + currentCount++; + if (currentCount > 4) + { + Console.WriteLine("WASM EXIT 0"); + } + """ + } + }); + } + + private async Task SetupCounterPage(IPage page) + { + await Task.Delay(500); + // Navigate to the Counter page + await page.Locator("text=Counter").ClickAsync(); + + // Verify we're on the Counter page + var txt = await page.Locator("p[role='status']").InnerHTMLAsync(); + Assert.Equal("Current count: 0", txt); + + var up = """ + globalThis.uploadTrace = async (filename, tracesPromise) => { + console.log(`${filename} typeof ${typeof tracesPromise} ${new Date().toISOString()}`); + + const traces = await tracesPromise; + + console.log(`${filename} typeof ${typeof traces} ${new Date().toISOString()}`); + + // concatenate the buffers into a single Uint8Array + const concatenated = new Uint8Array(traces.reduce((acc, curr) => acc + curr.byteLength, 0)); + let offset = 0; + for (const trace of traces) { + concatenated.set(new Uint8Array(trace), offset); + offset += trace.byteLength; + } + + console.log(`File ${filename} size to upload: ${concatenated.byteLength} bytes ${new Date().toISOString()}`); + await fetch(`/upload/${filename}`, { + headers: { + 'Content-Type': 'application/octet-stream' + }, + method: 'POST', + body: concatenated + }); + console.log(`File uploaded successfully`); + }; + console.log(`globalThis.uploadTrace defined ${new Date().toISOString()}`); + """; + await page.EvaluateAsync(up); + } + + private async Task ClickAndCollect(IPage page) + { + // Click the button a few times + for (int i = 0; i < 5; i++) + { + await page.Locator("text=\"Click me\"").ClickAsync(); + await Task.Delay(10); + } + await Task.Delay(1000); + await page.EvaluateAsync(@"globalThis.upload;"); + + var txt2 = await page.Locator("p[role='status']").InnerHTMLAsync(); + Assert.NotEqual("Current count: 0", txt2); + } +} diff --git a/src/runtime/src/mono/wasm/Wasm.Build.Tests/BrowserStructures/RunOptions.cs b/src/runtime/src/mono/wasm/Wasm.Build.Tests/BrowserStructures/RunOptions.cs index df379e86db3..c3c7984af63 100644 --- a/src/runtime/src/mono/wasm/Wasm.Build.Tests/BrowserStructures/RunOptions.cs +++ b/src/runtime/src/mono/wasm/Wasm.Build.Tests/BrowserStructures/RunOptions.cs @@ -27,6 +27,7 @@ public abstract record RunOptions string Locale = "en-US", int? ExpectedExitCode = 0, string CustomBundleDir = "", + int? TimeoutSeconds = 10, Func? ExecuteAfterLoaded = null ); diff --git a/src/runtime/src/mono/wasm/Wasm.Build.Tests/Templates/WasmTemplateTestsBase.cs b/src/runtime/src/mono/wasm/Wasm.Build.Tests/Templates/WasmTemplateTestsBase.cs index 1196d351cf3..5288275ce6a 100644 --- a/src/runtime/src/mono/wasm/Wasm.Build.Tests/Templates/WasmTemplateTestsBase.cs +++ b/src/runtime/src/mono/wasm/Wasm.Build.Tests/Templates/WasmTemplateTestsBase.cs @@ -335,7 +335,7 @@ private async Task BrowserRunTest(string runArgs, await blazorOp.Test(page); _testOutput.WriteLine($"Waiting for additional 10secs to see if any errors are reported"); - int exitCode = await runner.WaitForExitMessageAsync(TimeSpan.FromSeconds(10)); + int exitCode = await runner.WaitForExitMessageAsync(TimeSpan.FromSeconds(runOptions.TimeoutSeconds ?? 10)); if (runOptions.ExpectedExitCode is not null && exitCode != runOptions.ExpectedExitCode) throw new Exception($"Expected exit code {runOptions.ExpectedExitCode} but got {exitCode}.\nconsoleOutput={string.Join("\n", consoleOutput)}"); diff --git a/src/runtime/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj b/src/runtime/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj index ac77377d096..f574a9deeaf 100644 --- a/src/runtime/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj +++ b/src/runtime/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj @@ -48,6 +48,7 @@ + diff --git a/src/runtime/src/mono/wasm/host/DevServer/DevServerStartup.cs b/src/runtime/src/mono/wasm/host/DevServer/DevServerStartup.cs index 5df2eabcaa8..975a27ade05 100644 --- a/src/runtime/src/mono/wasm/host/DevServer/DevServerStartup.cs +++ b/src/runtime/src/mono/wasm/host/DevServer/DevServerStartup.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.IO; using System.Net.WebSockets; using System.Threading.Tasks; @@ -115,6 +116,58 @@ public static void Configure(IApplicationBuilder app, IOptions } } }); + + // Add general-purpose file upload endpoint when DEVSERVER_UPLOAD_PATH is set + string? fileUploadPath = Environment.GetEnvironmentVariable("DEVSERVER_UPLOAD_PATH"); + if (!string.IsNullOrEmpty(fileUploadPath)) + { + // Ensure the upload directory exists + if (!Directory.Exists(fileUploadPath)) + { + Directory.CreateDirectory(fileUploadPath!); + } + + // Route with filename parameter + endpoints.MapPost("/upload/{filename}", async context => + { + try + { + // Get the filename from the route + var routeValues = context.Request.RouteValues; + string? rawFileName = routeValues["filename"]?.ToString(); + + // Generate a unique name if none provided + if (string.IsNullOrEmpty(rawFileName)) + { + rawFileName = $"upload_{Guid.NewGuid():N}"; + } + + // Sanitize filename - IMPORTANT: Only use GetFileName to strip any path components + // This prevents directory traversal attacks like "../../../etc/passwd" + string fileName = Path.GetFileName(rawFileName); + + if (string.IsNullOrEmpty(fileName)) + { + fileName = $"upload_{Guid.NewGuid():N}"; + } + + string filePath = Path.Combine(fileUploadPath!, fileName); + + using (var outputStream = new FileStream(filePath, FileMode.Create)) + { + await context.Request.Body.CopyToAsync(outputStream); + } + + await context.Response.WriteAsync($"File saved to {filePath}"); + } + catch (Exception ex) + { + context.Response.StatusCode = 500; + await context.Response.WriteAsync($"Error processing upload: {ex.Message}"); + } + }); + } + }); ServerURLsProvider.ResolveServerUrlsOnApplicationStarted(app, logger, applicationLifetime, realUrlsAvailableTcs, "/_framework/debug"); diff --git a/src/runtime/src/mono/wasm/testassets/BlazorBasicTestApp/App/BlazorBasicTestApp.csproj b/src/runtime/src/mono/wasm/testassets/BlazorBasicTestApp/App/BlazorBasicTestApp.csproj index 42e84162d96..a76fd751306 100644 --- a/src/runtime/src/mono/wasm/testassets/BlazorBasicTestApp/App/BlazorBasicTestApp.csproj +++ b/src/runtime/src/mono/wasm/testassets/BlazorBasicTestApp/App/BlazorBasicTestApp.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/runtime/src/mono/wasm/testassets/BlazorBasicTestApp/App/BlazorBasicTestApp.runtimeconfig.json b/src/runtime/src/mono/wasm/testassets/BlazorBasicTestApp/App/BlazorBasicTestApp.runtimeconfig.json new file mode 100644 index 00000000000..2e204d1cb2f --- /dev/null +++ b/src/runtime/src/mono/wasm/testassets/BlazorBasicTestApp/App/BlazorBasicTestApp.runtimeconfig.json @@ -0,0 +1,13 @@ +{ + "runtimeOptions": { + "wasmHostProperties": { + "perHostConfig": [ + { + "name": "browser", + "html-path": "index.html", + "Host": "browser" + } + ] + } + } +} \ No newline at end of file diff --git a/src/runtime/src/native/external/zlib-ng-version.txt b/src/runtime/src/native/external/zlib-ng-version.txt index a892ef3e523..198e16578e1 100644 --- a/src/runtime/src/native/external/zlib-ng-version.txt +++ b/src/runtime/src/native/external/zlib-ng-version.txt @@ -1,15 +1,12 @@ -v2.2.1 -d54e3769be0c522015b784eca2af258b1c026107 +v2.2.4 +860e4cff7917d93f54f5d7f0bc1d0e8b1a3cb988 -https://github.com/zlib-ng/zlib-ng/releases/tag/2.2.1 +https://github.com/zlib-ng/zlib-ng/releases/tag/2.2.4 We have removed the following folders from our local copy as these files are not needed for our compilation: +- zlib-ng/.github/ - zlib-ng/doc/ - zlib-ng/test/ - zlib-ng/arch/s390/self-hosted-builder/ -Also, if the next version does not yet contain the fixes included in 12bc7edc73308f017ec40c6b2db694a6e3490ac2, cherry-pick it as a patch. - -Apply https://github.com/zlib-ng/zlib-ng/pull/1812 -Apply https://github.com/zlib-ng/zlib-ng/pull/1853 diff --git a/src/runtime/src/native/external/zlib-ng.cmake b/src/runtime/src/native/external/zlib-ng.cmake index b2338d682e1..ddc7825af7f 100644 --- a/src/runtime/src/native/external/zlib-ng.cmake +++ b/src/runtime/src/native/external/zlib-ng.cmake @@ -31,6 +31,13 @@ if (CLR_CMAKE_TARGET_BROWSER OR CLR_CMAKE_TARGET_WASI) endif() endif() +if (MSVC) + #zlib-ng sets /utf-8 which clashes with /source-charset:utf-8 that we set centrally + get_directory_property(dirCompileOptions COMPILE_OPTIONS) + string(REPLACE "/source-charset:utf-8" "" dirCompileOptions "${dirCompileOptions}") + set_directory_properties(PROPERTIES COMPILE_OPTIONS "${dirCompileOptions}") +endif() + set(BUILD_SHARED_LIBS OFF) # Shared libraries aren't supported in wasm set(SKIP_INSTALL_ALL ON) FetchContent_MakeAvailable(fetchzlibng) diff --git a/src/runtime/src/native/external/zlib-ng/.gitattributes b/src/runtime/src/native/external/zlib-ng/.gitattributes index ac21ec4592d..38fd673cc9e 100644 --- a/src/runtime/src/native/external/zlib-ng/.gitattributes +++ b/src/runtime/src/native/external/zlib-ng/.gitattributes @@ -1,8 +1,9 @@ -* text=auto -*.abi text eol=lf -*.c text -*.h text -*.sh text eol=lf +# By default, enforce LF line-endings on all files that are not considered binary files! +* text=auto eol=lf + crc32_braid_tbl.h hooks-max-size=1000000 -Makefile text -configure text eol=lf + +# Don't export git/github-related files in tar/zip archives +/.github export-ignore +.gitattributes export-ignore +.gitignore export-ignore diff --git a/src/runtime/src/native/external/zlib-ng/.gitignore b/src/runtime/src/native/external/zlib-ng/.gitignore index cd4a9c3feeb..71b9e0747f8 100644 --- a/src/runtime/src/native/external/zlib-ng/.gitignore +++ b/src/runtime/src/native/external/zlib-ng/.gitignore @@ -27,6 +27,9 @@ /switchlevels /zlib.pc /zlib-ng.pc +/zconf-ng.h.included +/zlib_name_mangling-ng.h +/zlib-ng.h .DS_Store *_fuzzer @@ -61,6 +64,7 @@ zconf-ng.h.cmakein ztest* /test/CTestTestfile.cmake /test/cmake_install.cmake +/.cache configure.log a.out @@ -72,6 +76,10 @@ a.out /arch/x86/Makefile .kdev4 *.kdev4 +Makefile.tmp +/makecrct +/maketrees +gzread.c /Debug /example.dir diff --git a/src/runtime/src/native/external/zlib-ng/CMakeLists.txt b/src/runtime/src/native/external/zlib-ng/CMakeLists.txt index 86e54f858bf..367413062fc 100644 --- a/src/runtime/src/native/external/zlib-ng/CMakeLists.txt +++ b/src/runtime/src/native/external/zlib-ng/CMakeLists.txt @@ -4,16 +4,20 @@ if(CMAKE_VERSION VERSION_LESS 3.12) endif() message(STATUS "Using CMake version ${CMAKE_VERSION}") +if(POLICY CMP0169) + cmake_policy(SET CMP0169 OLD) # CMake 3.30: call FetchContent_Populate() with just the name of a dependency +endif() + # If not specified on the command line, enable C11 as the default # Configuration items that affect the global compiler environment standards # should be issued before the "project" command. -if(NOT CMAKE_C_STANDARD) +if(NOT DEFINED CMAKE_C_STANDARD) set(CMAKE_C_STANDARD 11) # The C standard whose features are requested to build this target endif() -if(NOT CMAKE_C_STANDARD_REQUIRED) +if(NOT DEFINED CMAKE_C_STANDARD_REQUIRED) set(CMAKE_C_STANDARD_REQUIRED ON) # Boolean describing whether the value of C_STANDARD is a requirement endif() -if(NOT CMAKE_C_EXTENSIONS) +if(NOT DEFINED CMAKE_C_EXTENSIONS) set(CMAKE_C_EXTENSIONS OFF) # Boolean specifying whether compiler specific extensions are requested endif() set(VALID_C_STANDARDS "99" "11") @@ -89,7 +93,6 @@ option(WITH_MAINTAINER_WARNINGS "Build with project maintainer warnings" OFF) option(WITH_CODE_COVERAGE "Enable code coverage reporting" OFF) option(WITH_INFLATE_STRICT "Build with strict inflate distance checking" OFF) option(WITH_INFLATE_ALLOW_INVALID_DIST "Build with zero fill for inflate invalid distances" OFF) -option(WITH_UNALIGNED "Support unaligned reads on platforms that support it" ON) set(ZLIB_SYMBOL_PREFIX "" CACHE STRING "Give this prefix to all publicly exported symbols. Useful when embedding into a larger library. @@ -102,7 +105,7 @@ set_property(CACHE WITH_SANITIZER PROPERTY STRINGS "Memory" "Address" "Undefined if(BASEARCH_ARM_FOUND) option(WITH_ACLE "Build with ACLE" ON) option(WITH_NEON "Build with NEON intrinsics" ON) - cmake_dependent_option(WITH_ARMV6 "Build with ARMv6 SIMD" ON "NOT ARCH STREQUAL \"aarch64\"" OFF) + cmake_dependent_option(WITH_ARMV6 "Build with ARMv6 SIMD" ON "NOT ARCH MATCHES \"aarch64\"" OFF) elseif(BASEARCH_PPC_FOUND) option(WITH_ALTIVEC "Build with AltiVec (VMX) optimisations for PowerPC" ON) option(WITH_POWER8 "Build with optimisations for POWER8" ON) @@ -143,7 +146,6 @@ mark_as_advanced(FORCE WITH_RVV WITH_INFLATE_STRICT WITH_INFLATE_ALLOW_INVALID_DIST - WITH_UNALIGNED INSTALL_UTILS ) @@ -189,9 +191,9 @@ elseif(MSVC) # (who'd use cmake from an IDE...) but checking for ICC before checking for MSVC should # avoid mistakes. # /Oi ? - set(WARNFLAGS /W3) + set(WARNFLAGS /W3 /w34242 /WX) set(WARNFLAGS_MAINTAINER /W4) - set(WARNFLAGS_DISABLE) + set(WARNFLAGS_DISABLE /wd4206 /wd4054 /wd4324) if(BASEARCH_ARM_FOUND) add_definitions(-D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE) if(NOT "${ARCH}" MATCHES "aarch64") @@ -299,7 +301,7 @@ if(NOT WITH_RUNTIME_CPU_DETECTION) message(STATUS "WARNING: Microsoft Visual Studio does not support compile time detection of CPU features for \"/arch\" before \"AVX\"") # Workaround for MSVC. By default MSVC does not define the __SSE*__ macros. # Fix it if AVX is enabled. - set(CMAKE_REQUIRED_FLAGS "${NATIVEFLAG}") + set(CMAKE_REQUIRED_FLAGS "${NATIVEFLAG} ${ZNOLTOFLAG}") check_c_source_compiles( "#ifndef __AVX__ # error \"AVX is not enabled.\" @@ -332,12 +334,6 @@ if(NOT WITH_NATIVE_INSTRUCTIONS) endforeach() endif() -# Set architecture alignment requirements -if(NOT WITH_UNALIGNED) - add_definitions(-DNO_UNALIGNED) - message(STATUS "Unaligned reads manually disabled") -endif() - # Apply warning compiler flags if(WITH_MAINTAINER_WARNINGS) add_compile_options(${WARNFLAGS} ${WARNFLAGS_MAINTAINER} ${WARNFLAGS_DISABLE}) @@ -356,6 +352,16 @@ if(NOT WITH_CODE_COVERAGE AND NOT MSVC AND NOT CMAKE_C_FLAGS MATCHES "([\\/\\-]O CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}") endif() +# Force Visual C++ to use UTF-8 +if(MSVC) + if (NOT CMAKE_C_FLAGS MATCHES "[\\/\\-]utf-8") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /utf-8") + endif() + if (NOT CMAKE_CXX_FLAGS MATCHES "[\\/\\-]utf-8") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /utf-8") + endif() +endif() + # # Check for standard/system includes # @@ -485,6 +491,7 @@ endif() # # Check for __builtin_assume_aligned(x,n) support in the compiler # +set(CMAKE_REQUIRED_FLAGS ${ZNOLTOFLAG}) check_c_source_compiles( "char *test(char *buffer) { char *abuffer = __builtin_assume_aligned(buffer,64); @@ -497,10 +504,12 @@ check_c_source_compiles( if(HAVE_BUILTIN_ASSUME_ALIGNED) add_definitions(-DHAVE_BUILTIN_ASSUME_ALIGNED) endif() +set(CMAKE_REQUIRED_FLAGS) # # check for __builtin_ctz() support in the compiler # +set(CMAKE_REQUIRED_FLAGS ${ZNOLTOFLAG}) check_c_source_compiles( "int main(void) { unsigned int zero = 0; @@ -513,10 +522,12 @@ check_c_source_compiles( if(HAVE_BUILTIN_CTZ) add_definitions(-DHAVE_BUILTIN_CTZ) endif() +set(CMAKE_REQUIRED_FLAGS) # # check for __builtin_ctzll() support in the compiler # +set(CMAKE_REQUIRED_FLAGS ${ZNOLTOFLAG}) check_c_source_compiles( "int main(void) { unsigned int zero = 0; @@ -529,6 +540,7 @@ check_c_source_compiles( if(HAVE_BUILTIN_CTZLL) add_definitions(-DHAVE_BUILTIN_CTZLL) endif() +set(CMAKE_REQUIRED_FLAGS) # # check for ptrdiff_t support @@ -965,8 +977,10 @@ if(WITH_OPTIM) add_definitions(-DX86_AVX512) list(APPEND AVX512_SRCS ${ARCHDIR}/adler32_avx512.c) add_feature_info(AVX512_ADLER32 1 "Support AVX512-accelerated adler32, using \"${AVX512FLAG}\"") - list(APPEND ZLIB_ARCH_SRCS ${AVX512_SRCS}) + list(APPEND AVX512_SRCS ${ARCHDIR}/chunkset_avx512.c) + add_feature_info(AVX512_CHUNKSET 1 "Support AVX512 optimized chunkset, using \"${AVX512FLAG}\"") list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/adler32_avx512_p.h) + list(APPEND ZLIB_ARCH_SRCS ${AVX512_SRCS}) set_property(SOURCE ${AVX512_SRCS} PROPERTY COMPILE_FLAGS "${AVX512FLAG} ${NOLTOFLAG}") else() set(WITH_AVX512 OFF) @@ -1066,6 +1080,9 @@ set(ZLIB_PUBLIC_HDRS ${CMAKE_CURRENT_BINARY_DIR}/zlib${SUFFIX}.h ) set(ZLIB_PRIVATE_HDRS + arch/generic/chunk_permute_table.h + arch/generic/compare256_p.h + arch/generic/generic_functions.h adler32_p.h chunkset_tpl.h compare256_rle.h @@ -1172,10 +1189,6 @@ if (ZLIB_COMPAT) endif() endif() -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${ARCHDIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/arch/generic) - foreach(ZLIB_INSTALL_LIBRARY ${ZLIB_INSTALL_LIBRARIES}) if(NOT ZLIB_COMPAT) target_compile_definitions(${ZLIB_INSTALL_LIBRARY} PUBLIC ZLIBNG_NATIVE_API) diff --git a/src/runtime/src/native/external/zlib-ng/Makefile.in b/src/runtime/src/native/external/zlib-ng/Makefile.in index 41f3e924553..1493124bbf4 100644 --- a/src/runtime/src/native/external/zlib-ng/Makefile.in +++ b/src/runtime/src/native/external/zlib-ng/Makefile.in @@ -30,7 +30,7 @@ LDSHARED=$(CC) LDSHAREDFLAGS=-shared LDVERSIONSCRIPT= -VER=2.2.1 +VER=2.2.4 VER1=2 STATICLIB=$(LIBNAME1).a diff --git a/src/runtime/src/native/external/zlib-ng/README.md b/src/runtime/src/native/external/zlib-ng/README.md index 411621b52ff..28aad7f1dc4 100644 --- a/src/runtime/src/native/external/zlib-ng/README.md +++ b/src/runtime/src/native/external/zlib-ng/README.md @@ -25,7 +25,7 @@ Features * Compare256 implementations using SSE2, AVX2, Neon, POWER9 & RVV * Inflate chunk copying using SSE2, SSSE3, AVX, Neon & VSX * Support for hardware-accelerated deflate using IBM Z DFLTCC -* Unaligned memory read/writes and large bit buffer improvements +* Safe unaligned memory read/writes and large bit buffer improvements * Includes improvements from Cloudflare and Intel forks * Configure, CMake, and NMake build system support * Comprehensive set of CMake unit tests @@ -213,7 +213,6 @@ Advanced Build Options | WITH_CRC32_VX | --without-crc32-vx | Build with vectorized CRC32 on IBM Z | ON | | WITH_DFLTCC_DEFLATE | --with-dfltcc-deflate | Build with DFLTCC intrinsics for compression on IBM Z | OFF | | WITH_DFLTCC_INFLATE | --with-dfltcc-inflate | Build with DFLTCC intrinsics for decompression on IBM Z | OFF | -| WITH_UNALIGNED | --without-unaligned | Allow optimizations that use unaligned reads if safe on current arch| ON | | WITH_INFLATE_STRICT | | Build with strict inflate distance checking | OFF | | WITH_INFLATE_ALLOW_INVALID_DIST | | Build with zero fill for inflate invalid distances | OFF | | INSTALL_UTILS | | Copy minigzip and minideflate during install | OFF | diff --git a/src/runtime/src/native/external/zlib-ng/adler32_fold.c b/src/runtime/src/native/external/zlib-ng/adler32_fold.c deleted file mode 100644 index e2f6f9ac7dd..00000000000 --- a/src/runtime/src/native/external/zlib-ng/adler32_fold.c +++ /dev/null @@ -1,16 +0,0 @@ -/* adler32_fold.c -- adler32 folding interface - * Copyright (C) 2022 Adam Stylinski - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include "zbuild.h" -#include "functable.h" -#include "adler32_fold.h" - -#include - -Z_INTERNAL uint32_t adler32_fold_copy_c(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) { - adler = functable.adler32(adler, src, len); - memcpy(dst, src, len); - return adler; -} diff --git a/src/runtime/src/native/external/zlib-ng/adler32_fold.h b/src/runtime/src/native/external/zlib-ng/adler32_fold.h deleted file mode 100644 index 20aa1c7400b..00000000000 --- a/src/runtime/src/native/external/zlib-ng/adler32_fold.h +++ /dev/null @@ -1,11 +0,0 @@ -/* adler32_fold.h -- adler32 folding interface - * Copyright (C) 2022 Adam Stylinski - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#ifndef ADLER32_FOLD_H_ -#define ADLER32_FOLD_H_ - -Z_INTERNAL uint32_t adler32_fold_copy_c(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len); - -#endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/arm/arm_functions.h b/src/runtime/src/native/external/zlib-ng/arch/arm/arm_functions.h index 61c682710a5..b256c37bff8 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/arm/arm_functions.h +++ b/src/runtime/src/native/external/zlib-ng/arch/arm/arm_functions.h @@ -8,7 +8,7 @@ #ifdef ARM_NEON uint32_t adler32_neon(uint32_t adler, const uint8_t *buf, size_t len); uint32_t chunksize_neon(void); -uint8_t* chunkmemset_safe_neon(uint8_t *out, unsigned dist, unsigned len, unsigned left); +uint8_t* chunkmemset_safe_neon(uint8_t *out, uint8_t *from, unsigned len, unsigned left); # ifdef HAVE_BUILTIN_CTZLL uint32_t compare256_neon(const uint8_t *src0, const uint8_t *src1); @@ -56,7 +56,7 @@ void slide_hash_armv6(deflate_state *s); # endif # endif // ARM - ACLE -# if defined(ARM_ACLE) && defined(__ARM_ACLE) && defined(__ARM_FEATURE_CRC32) +# if defined(ARM_ACLE) && (defined(__ARM_ACLE) || defined(__ARM_FEATURE_CRC32)) # undef native_crc32 # define native_crc32 crc32_acle # endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/arm/chunkset_neon.c b/src/runtime/src/native/external/zlib-ng/arch/arm/chunkset_neon.c index 1c49ef56123..68c9fef699b 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/arm/chunkset_neon.c +++ b/src/runtime/src/native/external/zlib-ng/arch/arm/chunkset_neon.c @@ -5,12 +5,11 @@ #ifdef ARM_NEON #include "neon_intrins.h" #include "zbuild.h" +#include "zmemory.h" #include "arch/generic/chunk_permute_table.h" typedef uint8x16_t chunk_t; -#define CHUNK_SIZE 16 - #define HAVE_CHUNKMEMSET_2 #define HAVE_CHUNKMEMSET_4 #define HAVE_CHUNKMEMSET_8 @@ -33,21 +32,15 @@ static const lut_rem_pair perm_idx_lut[13] = { }; static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) { - uint16_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = vreinterpretq_u8_u16(vdupq_n_u16(tmp)); + *chunk = vreinterpretq_u8_u16(vdupq_n_u16(zng_memread_2(from))); } static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) { - uint32_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = vreinterpretq_u8_u32(vdupq_n_u32(tmp)); + *chunk = vreinterpretq_u8_u32(vdupq_n_u32(zng_memread_4(from))); } static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) { - uint64_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = vreinterpretq_u8_u64(vdupq_n_u64(tmp)); + *chunk = vreinterpretq_u8_u64(vdupq_n_u64(zng_memread_8(from))); } #define CHUNKSIZE chunksize_neon @@ -84,7 +77,9 @@ static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t a = vld1_u8(buf); b = vld1_u8(buf + 8); ret0 = vtbl1_u8(a, perm_vec0); - uint8x8x2_t ab = {{a, b}}; + uint8x8x2_t ab; + ab.val[0] = a; + ab.val[1] = b; ret1 = vtbl2_u8(ab, perm_vec1); return vcombine_u8(ret0, ret1); #endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/arm/compare256_neon.c b/src/runtime/src/native/external/zlib-ng/arch/arm/compare256_neon.c index 87d14c89c09..3d05152f348 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/arm/compare256_neon.c +++ b/src/runtime/src/native/external/zlib-ng/arch/arm/compare256_neon.c @@ -4,7 +4,7 @@ */ #include "zbuild.h" -#include "zutil_p.h" +#include "zmemory.h" #include "deflate.h" #include "fallback_builtins.h" diff --git a/src/runtime/src/native/external/zlib-ng/arch/arm/crc32_acle.c b/src/runtime/src/native/external/zlib-ng/arch/arm/crc32_acle.c index 116bcab1c23..a3a1fefc03b 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/arm/crc32_acle.c +++ b/src/runtime/src/native/external/zlib-ng/arch/arm/crc32_acle.c @@ -11,9 +11,9 @@ Z_INTERNAL Z_TARGET_CRC uint32_t crc32_acle(uint32_t crc, const uint8_t *buf, size_t len) { Z_REGISTER uint32_t c; - Z_REGISTER const uint16_t *buf2; - Z_REGISTER const uint32_t *buf4; - Z_REGISTER const uint64_t *buf8; + Z_REGISTER uint16_t buf2; + Z_REGISTER uint32_t buf4; + Z_REGISTER uint64_t buf8; c = ~crc; @@ -29,45 +29,43 @@ Z_INTERNAL Z_TARGET_CRC uint32_t crc32_acle(uint32_t crc, const uint8_t *buf, si len--; } - if ((len >= sizeof(uint16_t)) && ((ptrdiff_t)buf & sizeof(uint16_t))) { - buf2 = (const uint16_t *) buf; - c = __crc32h(c, *buf2++); + if ((len >= sizeof(uint16_t)) && ((ptrdiff_t)buf & (sizeof(uint32_t) - 1))) { + buf2 = *((uint16_t*)buf); + c = __crc32h(c, buf2); + buf += sizeof(uint16_t); len -= sizeof(uint16_t); - buf4 = (const uint32_t *) buf2; - } else { - buf4 = (const uint32_t *) buf; } - if ((len >= sizeof(uint32_t)) && ((ptrdiff_t)buf & sizeof(uint32_t))) { - c = __crc32w(c, *buf4++); + if ((len >= sizeof(uint32_t)) && ((ptrdiff_t)buf & (sizeof(uint64_t) - 1))) { + buf4 = *((uint32_t*)buf); + c = __crc32w(c, buf4); len -= sizeof(uint32_t); + buf += sizeof(uint32_t); } - buf8 = (const uint64_t *) buf4; - } else { - buf8 = (const uint64_t *) buf; } while (len >= sizeof(uint64_t)) { - c = __crc32d(c, *buf8++); + buf8 = *((uint64_t*)buf); + c = __crc32d(c, buf8); len -= sizeof(uint64_t); + buf += sizeof(uint64_t); } if (len >= sizeof(uint32_t)) { - buf4 = (const uint32_t *) buf8; - c = __crc32w(c, *buf4++); + buf4 = *((uint32_t*)buf); + c = __crc32w(c, buf4); len -= sizeof(uint32_t); - buf2 = (const uint16_t *) buf4; - } else { - buf2 = (const uint16_t *) buf8; + buf += sizeof(uint32_t); } if (len >= sizeof(uint16_t)) { - c = __crc32h(c, *buf2++); + buf2 = *((uint16_t*)buf); + c = __crc32h(c, buf2); len -= sizeof(uint16_t); + buf += sizeof(uint16_t); } - buf = (const unsigned char *) buf2; if (len) { c = __crc32b(c, *buf); } diff --git a/src/runtime/src/native/external/zlib-ng/arch/arm/insert_string_acle.c b/src/runtime/src/native/external/zlib-ng/arch/arm/insert_string_acle.c deleted file mode 100644 index aa8385c712e..00000000000 --- a/src/runtime/src/native/external/zlib-ng/arch/arm/insert_string_acle.c +++ /dev/null @@ -1,24 +0,0 @@ -/* insert_string_acle.c -- insert_string integer hash variant using ACLE's CRC instructions - * - * Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - * - */ - -#ifdef ARM_ACLE -#include "acle_intrins.h" -#include "../../zbuild.h" -#include "../../deflate.h" - -#define HASH_CALC(s, h, val) \ - h = __crc32w(0, val) - -#define HASH_CALC_VAR h -#define HASH_CALC_VAR_INIT uint32_t h = 0 - -#define UPDATE_HASH Z_TARGET_CRC update_hash_acle -#define INSERT_STRING Z_TARGET_CRC insert_string_acle -#define QUICK_INSERT_STRING Z_TARGET_CRC quick_insert_string_acle - -#include "../../insert_string_tpl.h" -#endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/arm/neon_intrins.h b/src/runtime/src/native/external/zlib-ng/arch/arm/neon_intrins.h index a9e99ec88a9..5dc242d5211 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/arm/neon_intrins.h +++ b/src/runtime/src/native/external/zlib-ng/arch/arm/neon_intrins.h @@ -36,20 +36,20 @@ # ifndef ARM_NEON_HASLD4 static inline uint16x8x4_t vld1q_u16_x4(uint16_t const *a) { - uint16x8x4_t ret = (uint16x8x4_t) {{ - vld1q_u16(a), - vld1q_u16(a+8), - vld1q_u16(a+16), - vld1q_u16(a+24)}}; + uint16x8x4_t ret; + ret.val[0] = vld1q_u16(a); + ret.val[1] = vld1q_u16(a+8); + ret.val[2] = vld1q_u16(a+16); + ret.val[3] = vld1q_u16(a+24); return ret; } static inline uint8x16x4_t vld1q_u8_x4(uint8_t const *a) { - uint8x16x4_t ret = (uint8x16x4_t) {{ - vld1q_u8(a), - vld1q_u8(a+16), - vld1q_u8(a+32), - vld1q_u8(a+48)}}; + uint8x16x4_t ret; + ret.val[0] = vld1q_u8(a); + ret.val[1] = vld1q_u8(a+16); + ret.val[2] = vld1q_u8(a+32); + ret.val[3] = vld1q_u8(a+48); return ret; } diff --git a/src/runtime/src/native/external/zlib-ng/arch/generic/Makefile.in b/src/runtime/src/native/external/zlib-ng/arch/generic/Makefile.in index 32c8242d026..2522d7d0cbf 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/generic/Makefile.in +++ b/src/runtime/src/native/external/zlib-ng/arch/generic/Makefile.in @@ -40,10 +40,10 @@ chunkset_c.o: $(SRCDIR)/chunkset_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/chunkset_tpl. chunkset_c.lo: $(SRCDIR)/chunkset_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/chunkset_tpl.h $(SRCTOP)/inffast_tpl.h $(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_c.c -compare256_c.o: $(SRCDIR)/compare256_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/zutil_p.h $(SRCTOP)/deflate.h $(SRCTOP)/fallback_builtins.h +compare256_c.o: $(SRCDIR)/compare256_c.c $(SRCTOP)/zbuild.h $(SRCDIR)/compare256_p.h $(SRCTOP)/zmemory.h $(SRCTOP)/deflate.h $(SRCTOP)/fallback_builtins.h $(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_c.c -compare256_c.lo: $(SRCDIR)/compare256_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/zutil_p.h $(SRCTOP)/deflate.h $(SRCTOP)/fallback_builtins.h +compare256_c.lo: $(SRCDIR)/compare256_c.c $(SRCTOP)/zbuild.h $(SRCDIR)/compare256_p.h $(SRCTOP)/zmemory.h $(SRCTOP)/deflate.h $(SRCTOP)/fallback_builtins.h $(CC) $(SFLAGS) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_c.c crc32_braid_c.o: $(SRCDIR)/crc32_braid_c.c $(SRCTOP)/zbuild.h $(SRCTOP)/crc32_braid_p.h $(SRCTOP)/crc32_braid_tbl.h diff --git a/src/runtime/src/native/external/zlib-ng/arch/generic/chunkset_c.c b/src/runtime/src/native/external/zlib-ng/arch/generic/chunkset_c.c index 7b2bb7ba367..0a585e6caaf 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/generic/chunkset_c.c +++ b/src/runtime/src/native/external/zlib-ng/arch/generic/chunkset_c.c @@ -3,6 +3,7 @@ */ #include "zbuild.h" +#include "zmemory.h" typedef uint64_t chunk_t; @@ -12,21 +13,20 @@ typedef uint64_t chunk_t; #define HAVE_CHUNKMEMSET_8 static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) { - uint8_t *dest = (uint8_t *)chunk; - memcpy(dest, from, sizeof(uint32_t)); - memcpy(dest+4, from, sizeof(uint32_t)); + uint32_t tmp = zng_memread_4(from); + *chunk = tmp | ((chunk_t)tmp << 32); } static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) { - memcpy(chunk, from, sizeof(uint64_t)); + *chunk = zng_memread_8(from); } static inline void loadchunk(uint8_t const *s, chunk_t *chunk) { - memcpy(chunk, (uint8_t *)s, sizeof(uint64_t)); + *chunk = zng_memread_8(s); } static inline void storechunk(uint8_t *out, chunk_t *chunk) { - memcpy(out, chunk, sizeof(uint64_t)); + zng_memwrite_8(out, *chunk); } #define CHUNKSIZE chunksize_c diff --git a/src/runtime/src/native/external/zlib-ng/arch/generic/compare256_c.c b/src/runtime/src/native/external/zlib-ng/arch/generic/compare256_c.c index 0c12cb3a4ec..ad535523a57 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/generic/compare256_c.c +++ b/src/runtime/src/native/external/zlib-ng/arch/generic/compare256_c.c @@ -4,178 +4,28 @@ */ #include "zbuild.h" -#include "zutil_p.h" -#include "deflate.h" -#include "fallback_builtins.h" - -/* ALIGNED, byte comparison */ -static inline uint32_t compare256_c_static(const uint8_t *src0, const uint8_t *src1) { - uint32_t len = 0; - - do { - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - } while (len < 256); - - return 256; -} +#include "compare256_p.h" + +// Set optimal COMPARE256 function variant +#if OPTIMAL_CMP == 8 +# define COMPARE256 compare256_8 +#elif defined(HAVE_BUILTIN_CTZLL) +# define COMPARE256 compare256_64 +#elif defined(HAVE_BUILTIN_CTZ) +# define COMPARE256 compare256_32 +#else +# define COMPARE256 compare256_16 +#endif Z_INTERNAL uint32_t compare256_c(const uint8_t *src0, const uint8_t *src1) { - return compare256_c_static(src0, src1); + return COMPARE256(src0, src1); } +// Generate longest_match_c #define LONGEST_MATCH longest_match_c -#define COMPARE256 compare256_c_static - #include "match_tpl.h" +// Generate longest_match_slow_c #define LONGEST_MATCH_SLOW #define LONGEST_MATCH longest_match_slow_c -#define COMPARE256 compare256_c_static - #include "match_tpl.h" - -#if defined(UNALIGNED_OK) && BYTE_ORDER == LITTLE_ENDIAN -/* 16-bit unaligned integer comparison */ -static inline uint32_t compare256_unaligned_16_static(const uint8_t *src0, const uint8_t *src1) { - uint32_t len = 0; - - do { - if (zng_memcmp_2(src0, src1) != 0) - return len + (*src0 == *src1); - src0 += 2, src1 += 2, len += 2; - - if (zng_memcmp_2(src0, src1) != 0) - return len + (*src0 == *src1); - src0 += 2, src1 += 2, len += 2; - - if (zng_memcmp_2(src0, src1) != 0) - return len + (*src0 == *src1); - src0 += 2, src1 += 2, len += 2; - - if (zng_memcmp_2(src0, src1) != 0) - return len + (*src0 == *src1); - src0 += 2, src1 += 2, len += 2; - } while (len < 256); - - return 256; -} - -Z_INTERNAL uint32_t compare256_unaligned_16(const uint8_t *src0, const uint8_t *src1) { - return compare256_unaligned_16_static(src0, src1); -} - -#define LONGEST_MATCH longest_match_unaligned_16 -#define COMPARE256 compare256_unaligned_16_static - -#include "match_tpl.h" - -#define LONGEST_MATCH_SLOW -#define LONGEST_MATCH longest_match_slow_unaligned_16 -#define COMPARE256 compare256_unaligned_16_static - -#include "match_tpl.h" - -#ifdef HAVE_BUILTIN_CTZ -/* 32-bit unaligned integer comparison */ -static inline uint32_t compare256_unaligned_32_static(const uint8_t *src0, const uint8_t *src1) { - uint32_t len = 0; - - do { - uint32_t sv, mv, diff; - - memcpy(&sv, src0, sizeof(sv)); - memcpy(&mv, src1, sizeof(mv)); - - diff = sv ^ mv; - if (diff) { - uint32_t match_byte = __builtin_ctz(diff) / 8; - return len + match_byte; - } - - src0 += 4, src1 += 4, len += 4; - } while (len < 256); - - return 256; -} - -Z_INTERNAL uint32_t compare256_unaligned_32(const uint8_t *src0, const uint8_t *src1) { - return compare256_unaligned_32_static(src0, src1); -} - -#define LONGEST_MATCH longest_match_unaligned_32 -#define COMPARE256 compare256_unaligned_32_static - -#include "match_tpl.h" - -#define LONGEST_MATCH_SLOW -#define LONGEST_MATCH longest_match_slow_unaligned_32 -#define COMPARE256 compare256_unaligned_32_static - -#include "match_tpl.h" - -#endif - -#if defined(UNALIGNED64_OK) && defined(HAVE_BUILTIN_CTZLL) -/* UNALIGNED64_OK, 64-bit integer comparison */ -static inline uint32_t compare256_unaligned_64_static(const uint8_t *src0, const uint8_t *src1) { - uint32_t len = 0; - - do { - uint64_t sv, mv, diff; - - memcpy(&sv, src0, sizeof(sv)); - memcpy(&mv, src1, sizeof(mv)); - - diff = sv ^ mv; - if (diff) { - uint64_t match_byte = __builtin_ctzll(diff) / 8; - return len + (uint32_t)match_byte; - } - - src0 += 8, src1 += 8, len += 8; - } while (len < 256); - - return 256; -} - -Z_INTERNAL uint32_t compare256_unaligned_64(const uint8_t *src0, const uint8_t *src1) { - return compare256_unaligned_64_static(src0, src1); -} - -#define LONGEST_MATCH longest_match_unaligned_64 -#define COMPARE256 compare256_unaligned_64_static - -#include "match_tpl.h" - -#define LONGEST_MATCH_SLOW -#define LONGEST_MATCH longest_match_slow_unaligned_64 -#define COMPARE256 compare256_unaligned_64_static - -#include "match_tpl.h" - -#endif - -#endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/generic/compare256_p.h b/src/runtime/src/native/external/zlib-ng/arch/generic/compare256_p.h new file mode 100644 index 00000000000..ac934841d4b --- /dev/null +++ b/src/runtime/src/native/external/zlib-ng/arch/generic/compare256_p.h @@ -0,0 +1,123 @@ +/* compare256_p.h -- 256 byte memory comparison with match length return + * Copyright (C) 2020 Nathan Moinvaziri + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zmemory.h" +#include "deflate.h" +#include "fallback_builtins.h" + +/* 8-bit integer comparison */ +static inline uint32_t compare256_8(const uint8_t *src0, const uint8_t *src1) { + uint32_t len = 0; + + do { + if (*src0 != *src1) + return len; + src0 += 1, src1 += 1, len += 1; + if (*src0 != *src1) + return len; + src0 += 1, src1 += 1, len += 1; + if (*src0 != *src1) + return len; + src0 += 1, src1 += 1, len += 1; + if (*src0 != *src1) + return len; + src0 += 1, src1 += 1, len += 1; + if (*src0 != *src1) + return len; + src0 += 1, src1 += 1, len += 1; + if (*src0 != *src1) + return len; + src0 += 1, src1 += 1, len += 1; + if (*src0 != *src1) + return len; + src0 += 1, src1 += 1, len += 1; + if (*src0 != *src1) + return len; + src0 += 1, src1 += 1, len += 1; + } while (len < 256); + + return 256; +} + +/* 16-bit integer comparison */ +static inline uint32_t compare256_16(const uint8_t *src0, const uint8_t *src1) { + uint32_t len = 0; + + do { + if (zng_memcmp_2(src0, src1) != 0) + return len + (*src0 == *src1); + src0 += 2, src1 += 2, len += 2; + + if (zng_memcmp_2(src0, src1) != 0) + return len + (*src0 == *src1); + src0 += 2, src1 += 2, len += 2; + + if (zng_memcmp_2(src0, src1) != 0) + return len + (*src0 == *src1); + src0 += 2, src1 += 2, len += 2; + + if (zng_memcmp_2(src0, src1) != 0) + return len + (*src0 == *src1); + src0 += 2, src1 += 2, len += 2; + } while (len < 256); + + return 256; +} + +#ifdef HAVE_BUILTIN_CTZ +/* 32-bit integer comparison */ +static inline uint32_t compare256_32(const uint8_t *src0, const uint8_t *src1) { + uint32_t len = 0; + + do { + uint32_t sv, mv, diff; + + sv = zng_memread_4(src0); + mv = zng_memread_4(src1); + + diff = sv ^ mv; + if (diff) { +# if BYTE_ORDER == LITTLE_ENDIAN + uint32_t match_byte = __builtin_ctz(diff) / 8; +# else + uint32_t match_byte = __builtin_clz(diff) / 8; +# endif + return len + match_byte; + } + + src0 += 4, src1 += 4, len += 4; + } while (len < 256); + + return 256; +} +#endif + +#ifdef HAVE_BUILTIN_CTZLL +/* 64-bit integer comparison */ +static inline uint32_t compare256_64(const uint8_t *src0, const uint8_t *src1) { + uint32_t len = 0; + + do { + uint64_t sv, mv, diff; + + sv = zng_memread_8(src0); + mv = zng_memread_8(src1); + + diff = sv ^ mv; + if (diff) { +# if BYTE_ORDER == LITTLE_ENDIAN + uint64_t match_byte = __builtin_ctzll(diff) / 8; +# else + uint64_t match_byte = __builtin_clzll(diff) / 8; +# endif + return len + (uint32_t)match_byte; + } + + src0 += 8, src1 += 8, len += 8; + } while (len < 256); + + return 256; +} +#endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/generic/generic_functions.h b/src/runtime/src/native/external/zlib-ng/arch/generic/generic_functions.h index 997dd4d01ee..b0366baeceb 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/generic/generic_functions.h +++ b/src/runtime/src/native/external/zlib-ng/arch/generic/generic_functions.h @@ -22,68 +22,19 @@ typedef uint32_t (*crc32_func)(uint32_t crc32, const uint8_t *buf, size_t len); uint32_t adler32_c(uint32_t adler, const uint8_t *buf, size_t len); uint32_t chunksize_c(void); -uint8_t* chunkmemset_safe_c(uint8_t *out, unsigned dist, unsigned len, unsigned left); +uint8_t* chunkmemset_safe_c(uint8_t *out, uint8_t *from, unsigned len, unsigned left); void inflate_fast_c(PREFIX3(stream) *strm, uint32_t start); uint32_t PREFIX(crc32_braid)(uint32_t crc, const uint8_t *buf, size_t len); uint32_t compare256_c(const uint8_t *src0, const uint8_t *src1); -#if defined(UNALIGNED_OK) && BYTE_ORDER == LITTLE_ENDIAN -uint32_t compare256_unaligned_16(const uint8_t *src0, const uint8_t *src1); -# ifdef HAVE_BUILTIN_CTZ - uint32_t compare256_unaligned_32(const uint8_t *src0, const uint8_t *src1); -# endif -# if defined(UNALIGNED64_OK) && defined(HAVE_BUILTIN_CTZLL) - uint32_t compare256_unaligned_64(const uint8_t *src0, const uint8_t *src1); -# endif -#endif typedef void (*slide_hash_func)(deflate_state *s); void slide_hash_c(deflate_state *s); uint32_t longest_match_c(deflate_state *const s, Pos cur_match); -# if defined(UNALIGNED_OK) && BYTE_ORDER == LITTLE_ENDIAN - uint32_t longest_match_unaligned_16(deflate_state *const s, Pos cur_match); -# ifdef HAVE_BUILTIN_CTZ - uint32_t longest_match_unaligned_32(deflate_state *const s, Pos cur_match); -# endif -# if defined(UNALIGNED64_OK) && defined(HAVE_BUILTIN_CTZLL) - uint32_t longest_match_unaligned_64(deflate_state *const s, Pos cur_match); -# endif -# endif - uint32_t longest_match_slow_c(deflate_state *const s, Pos cur_match); -# if defined(UNALIGNED_OK) && BYTE_ORDER == LITTLE_ENDIAN - uint32_t longest_match_slow_unaligned_16(deflate_state *const s, Pos cur_match); - uint32_t longest_match_slow_unaligned_32(deflate_state *const s, Pos cur_match); -# ifdef UNALIGNED64_OK - uint32_t longest_match_slow_unaligned_64(deflate_state *const s, Pos cur_match); -# endif -# endif - - -// Select generic implementation for longest_match, longest_match_slow, longest_match_slow functions. -#if defined(UNALIGNED_OK) && BYTE_ORDER == LITTLE_ENDIAN -# if defined(UNALIGNED64_OK) && defined(HAVE_BUILTIN_CTZLL) -# define longest_match_generic longest_match_unaligned_64 -# define longest_match_slow_generic longest_match_slow_unaligned_64 -# define compare256_generic compare256_unaligned_64 -# elif defined(HAVE_BUILTIN_CTZ) -# define longest_match_generic longest_match_unaligned_32 -# define longest_match_slow_generic longest_match_slow_unaligned_32 -# define compare256_generic compare256_unaligned_32 -# else -# define longest_match_generic longest_match_unaligned_16 -# define longest_match_slow_generic longest_match_slow_unaligned_16 -# define compare256_generic compare256_unaligned_16 -# endif -#else -# define longest_match_generic longest_match_c -# define longest_match_slow_generic longest_match_slow_c -# define compare256_generic compare256_c -#endif - #ifdef DISABLE_RUNTIME_CPU_DETECTION // Generic code @@ -98,9 +49,9 @@ uint32_t longest_match_slow_c(deflate_state *const s, Pos cur_match); # define native_crc32_fold_reset crc32_fold_reset_c # define native_inflate_fast inflate_fast_c # define native_slide_hash slide_hash_c -# define native_longest_match longest_match_generic -# define native_longest_match_slow longest_match_slow_generic -# define native_compare256 compare256_generic +# define native_longest_match longest_match_c +# define native_longest_match_slow longest_match_slow_c +# define native_compare256 compare256_c #endif #endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/power/chunkset_power8.c b/src/runtime/src/native/external/zlib-ng/arch/power/chunkset_power8.c index aef19732736..673fe0e1128 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/power/chunkset_power8.c +++ b/src/runtime/src/native/external/zlib-ng/arch/power/chunkset_power8.c @@ -5,6 +5,7 @@ #ifdef POWER8_VSX #include #include "zbuild.h" +#include "zmemory.h" typedef vector unsigned char chunk_t; @@ -15,21 +16,15 @@ typedef vector unsigned char chunk_t; #define HAVE_CHUNKMEMSET_8 static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) { - uint16_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = (vector unsigned char)vec_splats(tmp); + *chunk = (vector unsigned char)vec_splats(zng_memread_2(from)); } static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) { - uint32_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = (vector unsigned char)vec_splats(tmp); + *chunk = (vector unsigned char)vec_splats(zng_memread_4(from)); } static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) { - uint64_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = (vector unsigned char)vec_splats((unsigned long long)tmp); + *chunk = (vector unsigned char)vec_splats((unsigned long long)zng_memread_8(from)); } static inline void loadchunk(uint8_t const *s, chunk_t *chunk) { diff --git a/src/runtime/src/native/external/zlib-ng/arch/power/compare256_power9.c b/src/runtime/src/native/external/zlib-ng/arch/power/compare256_power9.c index c8be498e4f6..2875719c47c 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/power/compare256_power9.c +++ b/src/runtime/src/native/external/zlib-ng/arch/power/compare256_power9.c @@ -6,7 +6,7 @@ #ifdef POWER9 #include #include "zbuild.h" -#include "zutil_p.h" +#include "zmemory.h" #include "deflate.h" #include "zendian.h" diff --git a/src/runtime/src/native/external/zlib-ng/arch/power/crc32_power8.c b/src/runtime/src/native/external/zlib-ng/arch/power/crc32_power8.c index 1cb5f299f3d..28c59b663c7 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/power/crc32_power8.c +++ b/src/runtime/src/native/external/zlib-ng/arch/power/crc32_power8.c @@ -32,9 +32,7 @@ #include "crc32_constants.h" #include "crc32_braid_tbl.h" -#if defined (__clang__) -#include "fallback_builtins.h" -#endif +#include "power_intrins.h" #define MAX_SIZE 32768 #define VMX_ALIGN 16 diff --git a/src/runtime/src/native/external/zlib-ng/arch/power/power_functions.h b/src/runtime/src/native/external/zlib-ng/arch/power/power_functions.h index cb6b7650eca..44d36af8342 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/power/power_functions.h +++ b/src/runtime/src/native/external/zlib-ng/arch/power/power_functions.h @@ -15,7 +15,7 @@ void slide_hash_vmx(deflate_state *s); #ifdef POWER8_VSX uint32_t adler32_power8(uint32_t adler, const uint8_t *buf, size_t len); uint32_t chunksize_power8(void); -uint8_t* chunkmemset_safe_power8(uint8_t *out, unsigned dist, unsigned len, unsigned left); +uint8_t* chunkmemset_safe_power8(uint8_t *out, uint8_t *from, unsigned len, unsigned left); uint32_t crc32_power8(uint32_t crc, const uint8_t *buf, size_t len); void slide_hash_power8(deflate_state *s); void inflate_fast_power8(PREFIX3(stream) *strm, uint32_t start); diff --git a/src/runtime/src/native/external/zlib-ng/arch/power/fallback_builtins.h b/src/runtime/src/native/external/zlib-ng/arch/power/power_intrins.h similarity index 91% rename from src/runtime/src/native/external/zlib-ng/arch/power/fallback_builtins.h rename to src/runtime/src/native/external/zlib-ng/arch/power/power_intrins.h index ed9584617b1..cf71986563a 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/power/fallback_builtins.h +++ b/src/runtime/src/native/external/zlib-ng/arch/power/power_intrins.h @@ -9,9 +9,10 @@ * For conditions of distribution and use, see copyright notice in zlib.h */ -#ifndef POWER_BUILTINS_H -#define POWER_BUILTINS_H +#ifndef POWER_INTRINS_H +#define POWER_INTRINS_H +#if defined (__clang__) /* * These stubs fix clang incompatibilities with GCC builtins. */ @@ -29,3 +30,5 @@ vec_ld(int __a, const __vector unsigned long long* __b) { } #endif + +#endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/riscv/adler32_rvv.c b/src/runtime/src/native/external/zlib-ng/arch/riscv/adler32_rvv.c index d0f9aaa567b..d822d75af60 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/riscv/adler32_rvv.c +++ b/src/runtime/src/native/external/zlib-ng/arch/riscv/adler32_rvv.c @@ -72,6 +72,7 @@ static inline uint32_t adler32_rvv_impl(uint32_t adler, uint8_t* restrict dst, c /* do modulo once each block of NMAX size */ if (++cnt >= nmax_limit) { v_adler32_prev_accu = __riscv_vremu_vx_u32m4(v_adler32_prev_accu, BASE, vl); + v_buf32_accu = __riscv_vremu_vx_u32m4(v_buf32_accu, BASE, vl); cnt = 0; } } @@ -99,16 +100,19 @@ static inline uint32_t adler32_rvv_impl(uint32_t adler, uint8_t* restrict dst, c vuint32m1_t v_sum2_sum = __riscv_vmv_s_x_u32m1(0, vl); v_sum2_sum = __riscv_vredsum_vs_u32m4_u32m1(v_sum32_accu, v_sum2_sum, vl); - uint32_t sum2_sum = __riscv_vmv_x_s_u32m1_u32(v_sum2_sum); + uint32_t sum2_sum = __riscv_vmv_x_s_u32m1_u32(v_sum2_sum) % BASE; - sum2 += (sum2_sum + adler * (len - left)); + sum2 += (sum2_sum + adler * ((len - left) % BASE)); vuint32m1_t v_adler_sum = __riscv_vmv_s_x_u32m1(0, vl); v_adler_sum = __riscv_vredsum_vs_u32m4_u32m1(v_buf32_accu, v_adler_sum, vl); - uint32_t adler_sum = __riscv_vmv_x_s_u32m1_u32(v_adler_sum); + uint32_t adler_sum = __riscv_vmv_x_s_u32m1_u32(v_adler_sum) % BASE; adler += adler_sum; + sum2 %= BASE; + adler %= BASE; + while (left--) { if (COPY) *dst++ = *src; adler += *src++; diff --git a/src/runtime/src/native/external/zlib-ng/arch/riscv/compare256_rvv.c b/src/runtime/src/native/external/zlib-ng/arch/riscv/compare256_rvv.c index 3d6c3e3aa5b..3ddb4db0803 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/riscv/compare256_rvv.c +++ b/src/runtime/src/native/external/zlib-ng/arch/riscv/compare256_rvv.c @@ -7,7 +7,7 @@ #ifdef RISCV_RVV #include "zbuild.h" -#include "zutil_p.h" +#include "zmemory.h" #include "deflate.h" #include "fallback_builtins.h" diff --git a/src/runtime/src/native/external/zlib-ng/arch/riscv/riscv_features.c b/src/runtime/src/native/external/zlib-ng/arch/riscv/riscv_features.c index b1f166d749b..f9957d19ccc 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/riscv/riscv_features.c +++ b/src/runtime/src/native/external/zlib-ng/arch/riscv/riscv_features.c @@ -14,7 +14,10 @@ int Z_INTERNAL is_kernel_version_greater_or_equal_to_6_5() { struct utsname buffer; - uname(&buffer); + if (uname(&buffer) == -1) { + // uname failed + return 0; + } int major, minor; if (sscanf(buffer.release, "%d.%d", &major, &minor) != 2) { @@ -49,4 +52,20 @@ void Z_INTERNAL riscv_check_features(struct riscv_cpu_features *features) { riscv_check_features_runtime(features); else riscv_check_features_compile_time(features); + if (features->has_rvv) { + size_t e8m1_vec_len; + intptr_t vtype_reg_val; + // Check that a vuint8m1_t vector is at least 16 bytes and that tail + // agnostic and mask agnostic mode are supported + // + __asm__ volatile( + "vsetvli %0, zero, e8, m1, ta, ma\n\t" + "csrr %1, vtype" + : "=r"(e8m1_vec_len), "=r"(vtype_reg_val)); + + // The RVV target is supported if the VILL bit of VTYPE (the MSB bit of + // VTYPE) is not set and the length of a vuint8m1_t vector is at least 16 + // bytes + features->has_rvv = (vtype_reg_val >= 0 && e8m1_vec_len >= 16); + } } diff --git a/src/runtime/src/native/external/zlib-ng/arch/riscv/riscv_functions.h b/src/runtime/src/native/external/zlib-ng/arch/riscv/riscv_functions.h index 015b2fbd75c..1792b9d259d 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/riscv/riscv_functions.h +++ b/src/runtime/src/native/external/zlib-ng/arch/riscv/riscv_functions.h @@ -13,7 +13,7 @@ uint32_t adler32_rvv(uint32_t adler, const uint8_t *buf, size_t len); uint32_t adler32_fold_copy_rvv(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len); uint32_t chunksize_rvv(void); -uint8_t* chunkmemset_safe_rvv(uint8_t *out, unsigned dist, unsigned len, unsigned left); +uint8_t* chunkmemset_safe_rvv(uint8_t *out, uint8_t *from, unsigned len, unsigned left); uint32_t compare256_rvv(const uint8_t *src0, const uint8_t *src1); uint32_t longest_match_rvv(deflate_state *const s, Pos cur_match); diff --git a/src/runtime/src/native/external/zlib-ng/arch/s390/README.md b/src/runtime/src/native/external/zlib-ng/arch/s390/README.md index 7b383cc9981..c56ffd7654d 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/s390/README.md +++ b/src/runtime/src/native/external/zlib-ng/arch/s390/README.md @@ -222,56 +222,44 @@ need for constantly changing the patch. ## Configuring the builder. ### Install prerequisites. - ``` sudo dnf install podman ``` -### Add actions-runner service. +### Create a config file, needs github personal access token. +Access token needs permissions; Repo Admin RW, Org Self-hosted runners RW. +For details, consult +https://docs.github.com/en/rest/actions/self-hosted-runners?apiVersion=2022-11-28#create-a-registration-token-for-a-repository +#### Create file /etc/actions-runner: ``` -sudo cp self-hosted-builder/actions-runner.service /etc/systemd/system/ -sudo systemctl daemon-reload +REPO=/ +PAT_TOKEN= ``` -### Create a config file, needs github personal access token. - +#### Set permissions on /etc/actions-runner: ``` -# Create file /etc/actions-runner -repo=/ -access_token= +chmod 600 /etc/actions-runner ``` -Access token should have the repo scope, consult -https://docs.github.com/en/rest/reference/actions#create-a-registration-token-for-a-repository -for details. +### Add actions-runner service. +``` +sudo cp self-hosted-builder/actions-runner.service /etc/systemd/system/ +sudo systemctl daemon-reload +``` ### Autostart actions-runner. - ``` $ sudo systemctl enable --now actions-runner ``` -## Rebuilding the container - -In order to update the `gaplib-actions-runner` podman container, e.g. to get the -latest OS security fixes, follow these steps: +### Add auto-rebuild cronjob +``` +sudo cp self-hosted-builder/actions-runner-rebuild.sh /etc/cron.weekly/ +chmod +x /etc/cron.weekly/actions-runner-rebuild.sh ``` -# Stop actions-runner service -sudo systemctl stop actions-runner - -# Delete old container -sudo podman container rm gaplib-actions-runner - -# Delete old image -sudo podman image rm localhost/zlib-ng/actions-runner - -# Build image -sudo podman build --squash -f Dockerfile.zlib-ng --tag zlib-ng/actions-runner --build-arg . - -# Build container -sudo podman create --name=gaplib-actions-runner --env-file=/etc/actions-runner --init --interactive --volume=actions-runner-temp:/home/actions-runner zlib-ng/actions-runner -# Start actions-runner service -sudo systemctl start actions-runner +## Building / Rebuilding the container +``` +sudo /etc/cron.weekly/actions-runner-rebuild.sh ``` diff --git a/src/runtime/src/native/external/zlib-ng/arch/s390/dfltcc_common.c b/src/runtime/src/native/external/zlib-ng/arch/s390/dfltcc_common.c deleted file mode 100644 index 78be7181146..00000000000 --- a/src/runtime/src/native/external/zlib-ng/arch/s390/dfltcc_common.c +++ /dev/null @@ -1,40 +0,0 @@ -/* dfltcc_deflate.c - IBM Z DEFLATE CONVERSION CALL general support. */ - -#include "zbuild.h" -#include "dfltcc_common.h" -#include "dfltcc_detail.h" - -/* - Memory management. - - DFLTCC requires parameter blocks and window to be aligned. zlib-ng allows - users to specify their own allocation functions, so using e.g. - `posix_memalign' is not an option. Thus, we overallocate and take the - aligned portion of the buffer. -*/ - -static const int PAGE_ALIGN = 0x1000; - -void Z_INTERNAL *PREFIX(dfltcc_alloc_window)(PREFIX3(streamp) strm, uInt items, uInt size) { - void *p; - void *w; - - /* To simplify freeing, we store the pointer to the allocated buffer right - * before the window. Note that DFLTCC always uses HB_SIZE bytes. - */ - p = ZALLOC(strm, sizeof(void *) + MAX(items * size, HB_SIZE) + PAGE_ALIGN, sizeof(unsigned char)); - if (p == NULL) - return NULL; - w = ALIGN_UP((char *)p + sizeof(void *), PAGE_ALIGN); - *(void **)((char *)w - sizeof(void *)) = p; - return w; -} - -void Z_INTERNAL PREFIX(dfltcc_copy_window)(void *dest, const void *src, size_t n) { - memcpy(dest, src, MAX(n, HB_SIZE)); -} - -void Z_INTERNAL PREFIX(dfltcc_free_window)(PREFIX3(streamp) strm, void *w) { - if (w) - ZFREE(strm, *(void **)((unsigned char *)w - sizeof(void *))); -} diff --git a/src/runtime/src/native/external/zlib-ng/arch/s390/s390_features.c b/src/runtime/src/native/external/zlib-ng/arch/s390/s390_features.c index 629025d5bb1..1f90a926c05 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/s390/s390_features.c +++ b/src/runtime/src/native/external/zlib-ng/arch/s390/s390_features.c @@ -6,7 +6,7 @@ #endif #ifndef HWCAP_S390_VXRS -#define HWCAP_S390_VXRS HWCAP_S390_VX +#define HWCAP_S390_VXRS (1 << 11) #endif void Z_INTERNAL s390_check_features(struct s390_cpu_features *features) { diff --git a/src/runtime/src/native/external/zlib-ng/arch/s390/s390_functions.h b/src/runtime/src/native/external/zlib-ng/arch/s390/s390_functions.h index e9c67978f0a..e6855cd7011 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/s390/s390_functions.h +++ b/src/runtime/src/native/external/zlib-ng/arch/s390/s390_functions.h @@ -7,8 +7,15 @@ #ifdef S390_CRC32_VX uint32_t crc32_s390_vx(uint32_t crc, const uint8_t *buf, size_t len); + +#ifdef __clang__ +# if ((__clang_major__ == 18) || (__clang_major__ == 19 && (__clang_minor__ < 1 || (__clang_minor__ == 1 && __clang_patchlevel__ < 2)))) +# error CRC32-VX optimizations are broken due to compiler bug in Clang versions: 18.0.0 <= clang_version < 19.1.2. \ + Either disable the zlib-ng CRC32-VX optimization, or switch to another compiler/compiler version. +# endif #endif +#endif #ifdef DISABLE_RUNTIME_CPU_DETECTION # if defined(S390_CRC32_VX) && defined(__zarch__) && __ARCH__ >= 11 && defined(__VX__) diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/Makefile.in b/src/runtime/src/native/external/zlib-ng/arch/x86/Makefile.in index c13cd179c0c..a797517df32 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/Makefile.in +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/Makefile.in @@ -8,9 +8,9 @@ SFLAGS= INCLUDES= SUFFIX= -AVX512FLAG=-mavx512f -mavx512dq -mavx512vl -mavx512bw -AVX512VNNIFLAG=-mavx512vnni -AVX2FLAG=-mavx2 +AVX512FLAG=-mavx512f -mavx512dq -mavx512vl -mavx512bw -mbmi2 +AVX512VNNIFLAG=-mavx512vnni -mbmi2 +AVX2FLAG=-mavx2 -mbmi2 SSE2FLAG=-msse2 SSSE3FLAG=-mssse3 SSE42FLAG=-msse4.2 @@ -31,6 +31,7 @@ all: \ adler32_sse42.o adler32_sse42.lo \ adler32_ssse3.o adler32_ssse3.lo \ chunkset_avx2.o chunkset_avx2.lo \ + chunkset_avx512.o chunkset_avx512.lo \ chunkset_sse2.o chunkset_sse2.lo \ chunkset_ssse3.o chunkset_ssse3.lo \ compare256_avx2.o compare256_avx2.lo \ @@ -52,6 +53,12 @@ chunkset_avx2.o: chunkset_avx2.lo: $(CC) $(SFLAGS) $(AVX2FLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_avx2.c +chunkset_avx512.o: + $(CC) $(CFLAGS) $(AVX512FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_avx512.c + +chunkset_avx512.lo: + $(CC) $(SFLAGS) $(AVX512FLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_avx512.c + chunkset_sse2.o: $(CC) $(CFLAGS) $(SSE2FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_sse2.c diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/adler32_avx512_p.h b/src/runtime/src/native/external/zlib-ng/arch/x86/adler32_avx512_p.h index 5b79d2ab6ee..742269053c0 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/adler32_avx512_p.h +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/adler32_avx512_p.h @@ -3,6 +3,17 @@ #include #include + +/* Written because Visual C++ toolchains before v142 have constant overflow in AVX512 intrinsic macros */ +#if defined(_MSC_VER) && !defined(_MM_K0_REG8) +# undef _mm512_extracti64x4_epi64 +# define _mm512_extracti64x4_epi64(v1, e1) _mm512_maskz_extracti64x4_epi64(UINT8_MAX, v1, e1) +# undef _mm512_set1_epi16 +# define _mm512_set1_epi16(e1) _mm512_maskz_set1_epi16(UINT32_MAX, e1) +# undef _mm512_maddubs_epi16 +# define _mm512_maddubs_epi16(v1, v2) _mm512_maskz_maddubs_epi16(UINT32_MAX, v1, v2) +#endif + /* Written because *_add_epi32(a) sets off ubsan */ static inline uint32_t _mm512_reduce_add_epu32(__m512i x) { __m256i a = _mm512_extracti64x4_epi64(x, 1); diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/avx2_tables.h b/src/runtime/src/native/external/zlib-ng/arch/x86/avx2_tables.h new file mode 100644 index 00000000000..50759993b98 --- /dev/null +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/avx2_tables.h @@ -0,0 +1,44 @@ +#ifndef _AVX2_TABLES_H +#define _AVX2_TABLES_H + +#include "../generic/chunk_permute_table.h" + +/* Populate don't cares so that this is a direct lookup (with some indirection into the permute table), because dist can + * never be 0 - 2, we'll start with an offset, subtracting 3 from the input */ +static const lut_rem_pair perm_idx_lut[29] = { + { 0, 2}, /* 3 */ + { 0, 0}, /* don't care */ + { 1 * 32, 2}, /* 5 */ + { 2 * 32, 2}, /* 6 */ + { 3 * 32, 4}, /* 7 */ + { 0 * 32, 0}, /* don't care */ + { 4 * 32, 5}, /* 9 */ + { 5 * 32, 22}, /* 10 */ + { 6 * 32, 21}, /* 11 */ + { 7 * 32, 20}, /* 12 */ + { 8 * 32, 6}, /* 13 */ + { 9 * 32, 4}, /* 14 */ + {10 * 32, 2}, /* 15 */ + { 0 * 32, 0}, /* don't care */ + {11 * 32, 15}, /* 17 */ + {11 * 32 + 16, 14}, /* 18 */ + {11 * 32 + 16 * 2, 13}, /* 19 */ + {11 * 32 + 16 * 3, 12}, /* 20 */ + {11 * 32 + 16 * 4, 11}, /* 21 */ + {11 * 32 + 16 * 5, 10}, /* 22 */ + {11 * 32 + 16 * 6, 9}, /* 23 */ + {11 * 32 + 16 * 7, 8}, /* 24 */ + {11 * 32 + 16 * 8, 7}, /* 25 */ + {11 * 32 + 16 * 9, 6}, /* 26 */ + {11 * 32 + 16 * 10, 5}, /* 27 */ + {11 * 32 + 16 * 11, 4}, /* 28 */ + {11 * 32 + 16 * 12, 3}, /* 29 */ + {11 * 32 + 16 * 13, 2}, /* 30 */ + {11 * 32 + 16 * 14, 1} /* 31 */ +}; + +static const uint16_t half_rem_vals[13] = { + 1, 0, 1, 4, 2, 0, 7, 6, 5, 4, 3, 2, 1 +}; + +#endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_avx2.c b/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_avx2.c index 70620b91542..c7f336fde7b 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_avx2.c +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_avx2.c @@ -2,70 +2,43 @@ * For conditions of distribution and use, see copyright notice in zlib.h */ #include "zbuild.h" +#include "zmemory.h" #ifdef X86_AVX2 +#include "avx2_tables.h" #include -#include "../generic/chunk_permute_table.h" +#include "x86_intrins.h" typedef __m256i chunk_t; - -#define CHUNK_SIZE 32 +typedef __m128i halfchunk_t; #define HAVE_CHUNKMEMSET_2 #define HAVE_CHUNKMEMSET_4 #define HAVE_CHUNKMEMSET_8 +#define HAVE_CHUNKMEMSET_16 #define HAVE_CHUNK_MAG - -/* Populate don't cares so that this is a direct lookup (with some indirection into the permute table), because dist can - * never be 0 - 2, we'll start with an offset, subtracting 3 from the input */ -static const lut_rem_pair perm_idx_lut[29] = { - { 0, 2}, /* 3 */ - { 0, 0}, /* don't care */ - { 1 * 32, 2}, /* 5 */ - { 2 * 32, 2}, /* 6 */ - { 3 * 32, 4}, /* 7 */ - { 0 * 32, 0}, /* don't care */ - { 4 * 32, 5}, /* 9 */ - { 5 * 32, 22}, /* 10 */ - { 6 * 32, 21}, /* 11 */ - { 7 * 32, 20}, /* 12 */ - { 8 * 32, 6}, /* 13 */ - { 9 * 32, 4}, /* 14 */ - {10 * 32, 2}, /* 15 */ - { 0 * 32, 0}, /* don't care */ - {11 * 32, 15}, /* 17 */ - {11 * 32 + 16, 14}, /* 18 */ - {11 * 32 + 16 * 2, 13}, /* 19 */ - {11 * 32 + 16 * 3, 12}, /* 20 */ - {11 * 32 + 16 * 4, 11}, /* 21 */ - {11 * 32 + 16 * 5, 10}, /* 22 */ - {11 * 32 + 16 * 6, 9}, /* 23 */ - {11 * 32 + 16 * 7, 8}, /* 24 */ - {11 * 32 + 16 * 8, 7}, /* 25 */ - {11 * 32 + 16 * 9, 6}, /* 26 */ - {11 * 32 + 16 * 10, 5}, /* 27 */ - {11 * 32 + 16 * 11, 4}, /* 28 */ - {11 * 32 + 16 * 12, 3}, /* 29 */ - {11 * 32 + 16 * 13, 2}, /* 30 */ - {11 * 32 + 16 * 14, 1} /* 31 */ -}; +#define HAVE_HALF_CHUNK static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) { - int16_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = _mm256_set1_epi16(tmp); + *chunk = _mm256_set1_epi16(zng_memread_2(from)); } static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) { - int32_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = _mm256_set1_epi32(tmp); + *chunk = _mm256_set1_epi32(zng_memread_4(from)); } static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) { - int64_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = _mm256_set1_epi64x(tmp); + *chunk = _mm256_set1_epi64x(zng_memread_8(from)); +} + +static inline void chunkmemset_16(uint8_t *from, chunk_t *chunk) { + /* See explanation in chunkset_avx512.c */ +#if defined(_MSC_VER) && _MSC_VER <= 1900 + halfchunk_t half = _mm_loadu_si128((__m128i*)from); + *chunk = _mm256_inserti128_si256(_mm256_castsi128_si256(half), half, 1); +#else + *chunk = _mm256_broadcastsi128_si256(_mm_loadu_si128((__m128i*)from)); +#endif } static inline void loadchunk(uint8_t const *s, chunk_t *chunk) { @@ -99,10 +72,7 @@ static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t perm_vec = _mm256_add_epi8(perm_vec, permute_xform); ret_vec = _mm256_inserti128_si256(_mm256_castsi128_si256(ret_vec0), ret_vec0, 1); ret_vec = _mm256_shuffle_epi8(ret_vec, perm_vec); - } else if (dist == 16) { - __m128i ret_vec0 = _mm_loadu_si128((__m128i*)buf); - return _mm256_inserti128_si256(_mm256_castsi128_si256(ret_vec0), ret_vec0, 1); - } else { + } else { __m128i ret_vec0 = _mm_loadu_si128((__m128i*)buf); __m128i ret_vec1 = _mm_loadu_si128((__m128i*)(buf + 16)); /* Take advantage of the fact that only the latter half of the 256 bit vector will actually differ */ @@ -118,6 +88,33 @@ static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t return ret_vec; } +static inline void loadhalfchunk(uint8_t const *s, halfchunk_t *chunk) { + *chunk = _mm_loadu_si128((__m128i *)s); +} + +static inline void storehalfchunk(uint8_t *out, halfchunk_t *chunk) { + _mm_storeu_si128((__m128i *)out, *chunk); +} + +static inline chunk_t halfchunk2whole(halfchunk_t *chunk) { + /* We zero extend mostly to appease some memory sanitizers. These bytes are ultimately + * unlikely to be actually written or read from */ + return _mm256_zextsi128_si256(*chunk); +} + +static inline halfchunk_t GET_HALFCHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t dist) { + lut_rem_pair lut_rem = perm_idx_lut[dist - 3]; + __m128i perm_vec, ret_vec; + __msan_unpoison(buf + dist, 16 - dist); + ret_vec = _mm_loadu_si128((__m128i*)buf); + *chunk_rem = half_rem_vals[dist - 3]; + + perm_vec = _mm_load_si128((__m128i*)(permute_table + lut_rem.idx)); + ret_vec = _mm_shuffle_epi8(ret_vec, perm_vec); + + return ret_vec; +} + #define CHUNKSIZE chunksize_avx2 #define CHUNKCOPY chunkcopy_avx2 #define CHUNKUNROLL chunkunroll_avx2 diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_avx512.c b/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_avx512.c new file mode 100644 index 00000000000..9d28d33d5ec --- /dev/null +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_avx512.c @@ -0,0 +1,182 @@ +/* chunkset_avx512.c -- AVX512 inline functions to copy small data chunks. + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#include "zbuild.h" +#include "zmemory.h" + +#ifdef X86_AVX512 + +#include "avx2_tables.h" +#include +#include "x86_intrins.h" + +typedef __m256i chunk_t; +typedef __m128i halfchunk_t; +typedef __mmask32 mask_t; +typedef __mmask16 halfmask_t; + +#define HAVE_CHUNKMEMSET_2 +#define HAVE_CHUNKMEMSET_4 +#define HAVE_CHUNKMEMSET_8 +#define HAVE_CHUNKMEMSET_16 +#define HAVE_CHUNK_MAG +#define HAVE_HALF_CHUNK +#define HAVE_MASKED_READWRITE +#define HAVE_CHUNKCOPY +#define HAVE_HALFCHUNKCOPY + +static inline halfmask_t gen_half_mask(unsigned len) { + return (halfmask_t)_bzhi_u32(0xFFFF, len); +} + +static inline mask_t gen_mask(unsigned len) { + return (mask_t)_bzhi_u32(0xFFFFFFFF, len); +} + +static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) { + *chunk = _mm256_set1_epi16(zng_memread_2(from)); +} + +static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) { + *chunk = _mm256_set1_epi32(zng_memread_4(from)); +} + +static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) { + *chunk = _mm256_set1_epi64x(zng_memread_8(from)); +} + +static inline void chunkmemset_16(uint8_t *from, chunk_t *chunk) { + /* Unfortunately there seems to be a compiler bug in Visual Studio 2015 where + * the load is dumped to the stack with an aligned move for this memory-register + * broadcast. The vbroadcasti128 instruction is 2 fewer cycles and this dump to + * stack doesn't exist if compiled with optimizations. For the sake of working + * properly in a debugger, let's take the 2 cycle penalty */ +#if defined(_MSC_VER) && _MSC_VER <= 1900 + halfchunk_t half = _mm_loadu_si128((__m128i*)from); + *chunk = _mm256_inserti128_si256(_mm256_castsi128_si256(half), half, 1); +#else + *chunk = _mm256_broadcastsi128_si256(_mm_loadu_si128((__m128i*)from)); +#endif +} + +static inline void loadchunk(uint8_t const *s, chunk_t *chunk) { + *chunk = _mm256_loadu_si256((__m256i *)s); +} + +static inline void storechunk(uint8_t *out, chunk_t *chunk) { + _mm256_storeu_si256((__m256i *)out, *chunk); +} + +static inline uint8_t* CHUNKCOPY(uint8_t *out, uint8_t const *from, unsigned len) { + Assert(len > 0, "chunkcopy should never have a length 0"); + + chunk_t chunk; + uint32_t rem = len % sizeof(chunk_t); + + if (len < sizeof(chunk_t)) { + mask_t rem_mask = gen_mask(rem); + chunk = _mm256_maskz_loadu_epi8(rem_mask, from); + _mm256_mask_storeu_epi8(out, rem_mask, chunk); + return out + rem; + } + + loadchunk(from, &chunk); + rem = (rem == 0) ? sizeof(chunk_t) : rem; + storechunk(out, &chunk); + out += rem; + from += rem; + len -= rem; + + while (len > 0) { + loadchunk(from, &chunk); + storechunk(out, &chunk); + out += sizeof(chunk_t); + from += sizeof(chunk_t); + len -= sizeof(chunk_t); + } + + return out; +} + +static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t dist) { + lut_rem_pair lut_rem = perm_idx_lut[dist - 3]; + __m256i ret_vec; + *chunk_rem = lut_rem.remval; + + /* See the AVX2 implementation for more detailed comments. This is that + some masked + * loads to avoid an out of bounds read on the heap */ + + if (dist < 16) { + const __m256i permute_xform = + _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16); + __m256i perm_vec = _mm256_load_si256((__m256i*)(permute_table+lut_rem.idx)); + halfmask_t load_mask = gen_half_mask(dist); + __m128i ret_vec0 = _mm_maskz_loadu_epi8(load_mask, buf); + perm_vec = _mm256_add_epi8(perm_vec, permute_xform); + ret_vec = _mm256_inserti128_si256(_mm256_castsi128_si256(ret_vec0), ret_vec0, 1); + ret_vec = _mm256_shuffle_epi8(ret_vec, perm_vec); + } else { + halfmask_t load_mask = gen_half_mask(dist - 16); + __m128i ret_vec0 = _mm_loadu_si128((__m128i*)buf); + __m128i ret_vec1 = _mm_maskz_loadu_epi8(load_mask, (__m128i*)(buf + 16)); + __m128i perm_vec1 = _mm_load_si128((__m128i*)(permute_table + lut_rem.idx)); + halfmask_t xlane_mask = _mm_cmp_epi8_mask(perm_vec1, _mm_set1_epi8(15), _MM_CMPINT_LE); + __m128i latter_half = _mm_mask_shuffle_epi8(ret_vec1, xlane_mask, ret_vec0, perm_vec1); + ret_vec = _mm256_inserti128_si256(_mm256_castsi128_si256(ret_vec0), latter_half, 1); + } + + return ret_vec; +} + +static inline void storehalfchunk(uint8_t *out, halfchunk_t *chunk) { + _mm_storeu_si128((__m128i *)out, *chunk); +} + +static inline chunk_t halfchunk2whole(halfchunk_t *chunk) { + /* We zero extend mostly to appease some memory sanitizers. These bytes are ultimately + * unlikely to be actually written or read from */ + return _mm256_zextsi128_si256(*chunk); +} + +static inline halfchunk_t GET_HALFCHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t dist) { + lut_rem_pair lut_rem = perm_idx_lut[dist - 3]; + __m128i perm_vec, ret_vec; + halfmask_t load_mask = gen_half_mask(dist); + ret_vec = _mm_maskz_loadu_epi8(load_mask, buf); + *chunk_rem = half_rem_vals[dist - 3]; + + perm_vec = _mm_load_si128((__m128i*)(permute_table + lut_rem.idx)); + ret_vec = _mm_shuffle_epi8(ret_vec, perm_vec); + + return ret_vec; +} + +static inline uint8_t* HALFCHUNKCOPY(uint8_t *out, uint8_t const *from, unsigned len) { + Assert(len > 0, "chunkcopy should never have a length 0"); + halfchunk_t chunk; + + uint32_t rem = len % sizeof(halfchunk_t); + if (rem == 0) { + rem = sizeof(halfchunk_t); + } + + halfmask_t rem_mask = gen_half_mask(rem); + chunk = _mm_maskz_loadu_epi8(rem_mask, from); + _mm_mask_storeu_epi8(out, rem_mask, chunk); + + return out + rem; +} + +#define CHUNKSIZE chunksize_avx512 +#define CHUNKUNROLL chunkunroll_avx512 +#define CHUNKMEMSET chunkmemset_avx512 +#define CHUNKMEMSET_SAFE chunkmemset_safe_avx512 + +#include "chunkset_tpl.h" + +#define INFLATE_FAST inflate_fast_avx512 + +#include "inffast_tpl.h" + +#endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_sse2.c b/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_sse2.c index c402c0ee18f..dcfe2c70d23 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_sse2.c +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_sse2.c @@ -3,34 +3,27 @@ */ #include "zbuild.h" +#include "zmemory.h" #ifdef X86_SSE2 #include typedef __m128i chunk_t; -#define CHUNK_SIZE 16 - #define HAVE_CHUNKMEMSET_2 #define HAVE_CHUNKMEMSET_4 #define HAVE_CHUNKMEMSET_8 static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) { - int16_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = _mm_set1_epi16(tmp); + *chunk = _mm_set1_epi16(zng_memread_2(from)); } static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) { - int32_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = _mm_set1_epi32(tmp); + *chunk = _mm_set1_epi32(zng_memread_4(from)); } static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) { - int64_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = _mm_set1_epi64x(tmp); + *chunk = _mm_set1_epi64x(zng_memread_8(from)); } static inline void loadchunk(uint8_t const *s, chunk_t *chunk) { diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_ssse3.c b/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_ssse3.c index 722ecd3d51e..75b698c61b0 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_ssse3.c +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/chunkset_ssse3.c @@ -3,6 +3,7 @@ */ #include "zbuild.h" +#include "zmemory.h" #if defined(X86_SSSE3) #include @@ -10,8 +11,6 @@ typedef __m128i chunk_t; -#define CHUNK_SIZE 16 - #define HAVE_CHUNKMEMSET_2 #define HAVE_CHUNKMEMSET_4 #define HAVE_CHUNKMEMSET_8 @@ -35,21 +34,15 @@ static const lut_rem_pair perm_idx_lut[13] = { static inline void chunkmemset_2(uint8_t *from, chunk_t *chunk) { - int16_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = _mm_set1_epi16(tmp); + *chunk = _mm_set1_epi16(zng_memread_2(from)); } static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) { - int32_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = _mm_set1_epi32(tmp); + *chunk = _mm_set1_epi32(zng_memread_4(from)); } static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) { - int64_t tmp; - memcpy(&tmp, from, sizeof(tmp)); - *chunk = _mm_set1_epi64x(tmp); + *chunk = _mm_set1_epi64x(zng_memread_8(from)); } static inline void loadchunk(uint8_t const *s, chunk_t *chunk) { diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/compare256_avx2.c b/src/runtime/src/native/external/zlib-ng/arch/x86/compare256_avx2.c index d2c835e4ee8..8a0213c3a62 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/compare256_avx2.c +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/compare256_avx2.c @@ -4,7 +4,7 @@ */ #include "zbuild.h" -#include "zutil_p.h" +#include "zmemory.h" #include "deflate.h" #include "fallback_builtins.h" diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/compare256_sse2.c b/src/runtime/src/native/external/zlib-ng/arch/x86/compare256_sse2.c index 216bb3a705c..25b65316a8b 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/compare256_sse2.c +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/compare256_sse2.c @@ -4,7 +4,7 @@ */ #include "zbuild.h" -#include "zutil_p.h" +#include "zmemory.h" #include "deflate.h" #include "fallback_builtins.h" diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/insert_string_sse42.c b/src/runtime/src/native/external/zlib-ng/arch/x86/insert_string_sse42.c deleted file mode 100644 index ae092a7e477..00000000000 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/insert_string_sse42.c +++ /dev/null @@ -1,24 +0,0 @@ -/* insert_string_sse42.c -- insert_string integer hash variant using SSE4.2's CRC instructions - * - * Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - * - */ - -#ifdef X86_SSE42 -#include "../../zbuild.h" -#include -#include "../../deflate.h" - -#define HASH_CALC(s, h, val)\ - h = _mm_crc32_u32(h, val) - -#define HASH_CALC_VAR h -#define HASH_CALC_VAR_INIT uint32_t h = 0 - -#define UPDATE_HASH update_hash_sse42 -#define INSERT_STRING insert_string_sse42 -#define QUICK_INSERT_STRING quick_insert_string_sse42 - -#include "../../insert_string_tpl.h" -#endif diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/x86_features.c b/src/runtime/src/native/external/zlib-ng/arch/x86/x86_features.c index 58cb4df341f..9491a007308 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/x86_features.c +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/x86_features.c @@ -97,6 +97,8 @@ void Z_INTERNAL x86_check_features(struct x86_cpu_features *features) { features->has_avx2 = ebx & 0x20; } + features->has_bmi2 = ebx & 0x8; + // check AVX512 bits if the OS supports saving ZMM registers if (features->has_os_save_zmm) { features->has_avx512f = ebx & 0x00010000; @@ -108,7 +110,7 @@ void Z_INTERNAL x86_check_features(struct x86_cpu_features *features) { features->has_avx512vl = ebx & 0x80000000; } features->has_avx512_common = features->has_avx512f && features->has_avx512dq && features->has_avx512bw \ - && features->has_avx512vl; + && features->has_avx512vl && features->has_bmi2; features->has_avx512vnni = ecx & 0x800; } } diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/x86_features.h b/src/runtime/src/native/external/zlib-ng/arch/x86/x86_features.h index 6daa5e38282..3901ad75bec 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/x86_features.h +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/x86_features.h @@ -14,6 +14,7 @@ struct x86_cpu_features { int has_avx512vl; int has_avx512_common; // Enabled when AVX512(F,DQ,BW,VL) are all enabled. int has_avx512vnni; + int has_bmi2; int has_sse2; int has_ssse3; int has_sse42; diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/x86_functions.h b/src/runtime/src/native/external/zlib-ng/arch/x86/x86_functions.h index 5aa9b317474..fc62daeae15 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/x86_functions.h +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/x86_functions.h @@ -8,7 +8,7 @@ #ifdef X86_SSE2 uint32_t chunksize_sse2(void); -uint8_t* chunkmemset_safe_sse2(uint8_t *out, unsigned dist, unsigned len, unsigned left); +uint8_t* chunkmemset_safe_sse2(uint8_t *out, uint8_t *from, unsigned len, unsigned left); # ifdef HAVE_BUILTIN_CTZ uint32_t compare256_sse2(const uint8_t *src0, const uint8_t *src1); @@ -21,7 +21,7 @@ uint8_t* chunkmemset_safe_sse2(uint8_t *out, unsigned dist, unsigned len, unsign #ifdef X86_SSSE3 uint32_t adler32_ssse3(uint32_t adler, const uint8_t *buf, size_t len); -uint8_t* chunkmemset_safe_ssse3(uint8_t *out, unsigned dist, unsigned len, unsigned left); +uint8_t* chunkmemset_safe_ssse3(uint8_t *out, uint8_t *from, unsigned len, unsigned left); void inflate_fast_ssse3(PREFIX3(stream) *strm, uint32_t start); #endif @@ -33,7 +33,7 @@ uint32_t adler32_fold_copy_sse42(uint32_t adler, uint8_t *dst, const uint8_t *sr uint32_t adler32_avx2(uint32_t adler, const uint8_t *buf, size_t len); uint32_t adler32_fold_copy_avx2(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len); uint32_t chunksize_avx2(void); -uint8_t* chunkmemset_safe_avx2(uint8_t *out, unsigned dist, unsigned len, unsigned left); +uint8_t* chunkmemset_safe_avx2(uint8_t *out, uint8_t *from, unsigned len, unsigned left); # ifdef HAVE_BUILTIN_CTZ uint32_t compare256_avx2(const uint8_t *src0, const uint8_t *src1); @@ -46,6 +46,9 @@ uint8_t* chunkmemset_safe_avx2(uint8_t *out, unsigned dist, unsigned len, unsign #ifdef X86_AVX512 uint32_t adler32_avx512(uint32_t adler, const uint8_t *buf, size_t len); uint32_t adler32_fold_copy_avx512(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len); +uint32_t chunksize_avx512(void); +uint8_t* chunkmemset_safe_avx512(uint8_t *out, uint8_t *from, unsigned len, unsigned left); +void inflate_fast_avx512(PREFIX3(stream)* strm, uint32_t start); #endif #ifdef X86_AVX512VNNI uint32_t adler32_avx512_vnni(uint32_t adler, const uint8_t *buf, size_t len); @@ -146,6 +149,12 @@ uint32_t crc32_vpclmulqdq(uint32_t crc32, const uint8_t *buf, size_t len); # define native_adler32 adler32_avx512 # undef native_adler32_fold_copy # define native_adler32_fold_copy adler32_fold_copy_avx512 +# undef native_chunkmemset_safe +# define native_chunkmemset_safe chunkmemset_safe_avx512 +# undef native_chunksize +# define native_chunksize chunksize_avx512 +# undef native_inflate_fast +# define native_inflate_fast inflate_fast_avx512 // X86 - AVX512 (VNNI) # if defined(X86_AVX512VNNI) && defined(__AVX512VNNI__) # undef native_adler32 diff --git a/src/runtime/src/native/external/zlib-ng/arch/x86/x86_intrins.h b/src/runtime/src/native/external/zlib-ng/arch/x86/x86_intrins.h index 0e596d18a14..a2ec0027c31 100644 --- a/src/runtime/src/native/external/zlib-ng/arch/x86/x86_intrins.h +++ b/src/runtime/src/native/external/zlib-ng/arch/x86/x86_intrins.h @@ -84,4 +84,9 @@ static inline __m512i _mm512_zextsi128_si512(__m128i a) { #endif // __AVX512F__ #endif // defined(_MSC_VER) && _MSC_VER < 1914 +/* Visual C++ toolchains before v142 have constant overflow in AVX512 intrinsics */ +#if defined(_MSC_VER) && defined(__AVX512F__) && !defined(_MM_K0_REG8) +# undef _mm512_extracti32x4_epi32 +# define _mm512_extracti32x4_epi32(v1, e1) _mm512_maskz_extracti32x4_epi32(UINT8_MAX, v1, e1) +#endif #endif // include guard X86_INTRINS_H diff --git a/src/runtime/src/native/external/zlib-ng/chunkset.c b/src/runtime/src/native/external/zlib-ng/chunkset.c deleted file mode 100644 index 7b2bb7ba367..00000000000 --- a/src/runtime/src/native/external/zlib-ng/chunkset.c +++ /dev/null @@ -1,42 +0,0 @@ -/* chunkset.c -- inline functions to copy small data chunks. - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include "zbuild.h" - -typedef uint64_t chunk_t; - -#define CHUNK_SIZE 8 - -#define HAVE_CHUNKMEMSET_4 -#define HAVE_CHUNKMEMSET_8 - -static inline void chunkmemset_4(uint8_t *from, chunk_t *chunk) { - uint8_t *dest = (uint8_t *)chunk; - memcpy(dest, from, sizeof(uint32_t)); - memcpy(dest+4, from, sizeof(uint32_t)); -} - -static inline void chunkmemset_8(uint8_t *from, chunk_t *chunk) { - memcpy(chunk, from, sizeof(uint64_t)); -} - -static inline void loadchunk(uint8_t const *s, chunk_t *chunk) { - memcpy(chunk, (uint8_t *)s, sizeof(uint64_t)); -} - -static inline void storechunk(uint8_t *out, chunk_t *chunk) { - memcpy(out, chunk, sizeof(uint64_t)); -} - -#define CHUNKSIZE chunksize_c -#define CHUNKCOPY chunkcopy_c -#define CHUNKUNROLL chunkunroll_c -#define CHUNKMEMSET chunkmemset_c -#define CHUNKMEMSET_SAFE chunkmemset_safe_c - -#include "chunkset_tpl.h" - -#define INFLATE_FAST inflate_fast_c - -#include "inffast_tpl.h" diff --git a/src/runtime/src/native/external/zlib-ng/chunkset_tpl.h b/src/runtime/src/native/external/zlib-ng/chunkset_tpl.h index f5cc5c04506..383b4d8f84b 100644 --- a/src/runtime/src/native/external/zlib-ng/chunkset_tpl.h +++ b/src/runtime/src/native/external/zlib-ng/chunkset_tpl.h @@ -5,10 +5,6 @@ #include "zbuild.h" #include -#if CHUNK_SIZE == 32 && defined(X86_SSSE3) -extern uint8_t* chunkmemset_ssse3(uint8_t *out, unsigned dist, unsigned len); -#endif - /* Returns the chunk size */ Z_INTERNAL uint32_t CHUNKSIZE(void) { return sizeof(chunk_t); @@ -73,50 +69,111 @@ static inline uint8_t* CHUNKUNROLL(uint8_t *out, unsigned *dist, unsigned *len) static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t dist) { /* This code takes string of length dist from "from" and repeats * it for as many times as can fit in a chunk_t (vector register) */ - uint32_t cpy_dist; - uint32_t bytes_remaining = sizeof(chunk_t); + uint64_t cpy_dist; + uint64_t bytes_remaining = sizeof(chunk_t); chunk_t chunk_load; uint8_t *cur_chunk = (uint8_t *)&chunk_load; while (bytes_remaining) { cpy_dist = MIN(dist, bytes_remaining); - memcpy(cur_chunk, buf, cpy_dist); + memcpy(cur_chunk, buf, (size_t)cpy_dist); bytes_remaining -= cpy_dist; cur_chunk += cpy_dist; /* This allows us to bypass an expensive integer division since we're effectively * counting in this loop, anyway */ - *chunk_rem = cpy_dist; + *chunk_rem = (uint32_t)cpy_dist; } return chunk_load; } #endif +#if defined(HAVE_HALF_CHUNK) && !defined(HAVE_HALFCHUNKCOPY) +static inline uint8_t* HALFCHUNKCOPY(uint8_t *out, uint8_t const *from, unsigned len) { + halfchunk_t chunk; + int32_t align = ((len - 1) % sizeof(halfchunk_t)) + 1; + loadhalfchunk(from, &chunk); + storehalfchunk(out, &chunk); + out += align; + from += align; + len -= align; + while (len > 0) { + loadhalfchunk(from, &chunk); + storehalfchunk(out, &chunk); + out += sizeof(halfchunk_t); + from += sizeof(halfchunk_t); + len -= sizeof(halfchunk_t); + } + return out; +} +#endif + /* Copy DIST bytes from OUT - DIST into OUT + DIST * k, for 0 <= k < LEN/DIST. Return OUT + LEN. */ -Z_INTERNAL uint8_t* CHUNKMEMSET(uint8_t *out, unsigned dist, unsigned len) { +static inline uint8_t* CHUNKMEMSET(uint8_t *out, uint8_t *from, unsigned len) { /* Debug performance related issues when len < sizeof(uint64_t): Assert(len >= sizeof(uint64_t), "chunkmemset should be called on larger chunks"); */ - Assert(dist > 0, "chunkmemset cannot have a distance 0"); - /* Only AVX2 */ -#if CHUNK_SIZE == 32 && defined(X86_SSSE3) - if (len <= 16) { - return chunkmemset_ssse3(out, dist, len); - } -#endif + Assert(from != out, "chunkmemset cannot have a distance 0"); - uint8_t *from = out - dist; + chunk_t chunk_load; + uint32_t chunk_mod = 0; + uint32_t adv_amount; + int64_t sdist = out - from; + uint64_t dist = llabs(sdist); + + /* We are supporting the case for when we are reading bytes from ahead in the buffer. + * We now have to handle this, though it wasn't _quite_ clear if this rare circumstance + * always needed to be handled here or if we're just now seeing it because we are + * dispatching to this function, more */ + if (sdist < 0 && dist < len) { +#ifdef HAVE_MASKED_READWRITE + /* We can still handle this case if we can mitigate over writing _and_ we + * fit the entirety of the copy length with one load */ + if (len <= sizeof(chunk_t)) { + /* Tempting to add a goto to the block below but hopefully most compilers + * collapse these identical code segments as one label to jump to */ + return CHUNKCOPY(out, from, len); + } +#endif + /* Here the memmove semantics match perfectly, as when this happens we are + * effectively sliding down the contents of memory by dist bytes */ + memmove(out, from, len); + return out + len; + } if (dist == 1) { memset(out, *from, len); return out + len; - } else if (dist > sizeof(chunk_t)) { - return CHUNKCOPY(out, out - dist, len); + } else if (dist >= sizeof(chunk_t)) { + return CHUNKCOPY(out, from, len); } - chunk_t chunk_load; - uint32_t chunk_mod = 0; + /* Only AVX2+ as there's 128 bit vectors and 256 bit. We allow for shorter vector + * lengths because they serve to allow more cases to fall into chunkcopy, as the + * distance of the shorter length is still deemed a safe distance. We rewrite this + * here rather than calling the ssse3 variant directly now because doing so required + * dispatching to another function and broke inlining for this function entirely. We + * also can merge an assert and some remainder peeling behavior into the same code blocks, + * making the code a little smaller. */ +#ifdef HAVE_HALF_CHUNK + if (len <= sizeof(halfchunk_t)) { + if (dist >= sizeof(halfchunk_t)) + return HALFCHUNKCOPY(out, from, len); + + if ((dist % 2) != 0 || dist == 6) { + halfchunk_t halfchunk_load = GET_HALFCHUNK_MAG(from, &chunk_mod, (unsigned)dist); + + if (len == sizeof(halfchunk_t)) { + storehalfchunk(out, &halfchunk_load); + len -= sizeof(halfchunk_t); + out += sizeof(halfchunk_t); + } + + chunk_load = halfchunk2whole(&halfchunk_load); + goto rem_bytes; + } + } +#endif - /* TODO: possibly build up a permutation table for this if not an even modulus */ #ifdef HAVE_CHUNKMEMSET_2 if (dist == 2) { chunkmemset_2(from, &chunk_load); @@ -130,36 +187,37 @@ Z_INTERNAL uint8_t* CHUNKMEMSET(uint8_t *out, unsigned dist, unsigned len) { #ifdef HAVE_CHUNKMEMSET_8 if (dist == 8) { chunkmemset_8(from, &chunk_load); - } else if (dist == sizeof(chunk_t)) { - loadchunk(from, &chunk_load); } else #endif - { - chunk_load = GET_CHUNK_MAG(from, &chunk_mod, dist); - } +#ifdef HAVE_CHUNKMEMSET_16 + if (dist == 16) { + chunkmemset_16(from, &chunk_load); + } else +#endif + chunk_load = GET_CHUNK_MAG(from, &chunk_mod, (unsigned)dist); - /* If we're lucky enough and dist happens to be an even modulus of our vector length, - * we can do two stores per loop iteration, which for most ISAs, especially x86, is beneficial */ - if (chunk_mod == 0) { - while (len >= (2 * sizeof(chunk_t))) { - storechunk(out, &chunk_load); - storechunk(out + sizeof(chunk_t), &chunk_load); - out += 2 * sizeof(chunk_t); - len -= 2 * sizeof(chunk_t); - } + adv_amount = sizeof(chunk_t) - chunk_mod; + + while (len >= (2 * sizeof(chunk_t))) { + storechunk(out, &chunk_load); + storechunk(out + adv_amount, &chunk_load); + out += 2 * adv_amount; + len -= 2 * adv_amount; } /* If we don't have a "dist" length that divides evenly into a vector * register, we can write the whole vector register but we need only * advance by the amount of the whole string that fits in our chunk_t. * If we do divide evenly into the vector length, adv_amount = chunk_t size*/ - uint32_t adv_amount = sizeof(chunk_t) - chunk_mod; while (len >= sizeof(chunk_t)) { storechunk(out, &chunk_load); len -= adv_amount; out += adv_amount; } +#ifdef HAVE_HALF_CHUNK +rem_bytes: +#endif if (len) { memcpy(out, &chunk_load, len); out += len; @@ -168,33 +226,58 @@ Z_INTERNAL uint8_t* CHUNKMEMSET(uint8_t *out, unsigned dist, unsigned len) { return out; } -Z_INTERNAL uint8_t* CHUNKMEMSET_SAFE(uint8_t *out, unsigned dist, unsigned len, unsigned left) { -#if !defined(UNALIGNED64_OK) -# if !defined(UNALIGNED_OK) +Z_INTERNAL uint8_t* CHUNKMEMSET_SAFE(uint8_t *out, uint8_t *from, unsigned len, unsigned left) { +#if OPTIMAL_CMP < 32 static const uint32_t align_mask = 7; -# else +#elif OPTIMAL_CMP == 32 static const uint32_t align_mask = 3; -# endif #endif len = MIN(len, left); - uint8_t *from = out - dist; -#if !defined(UNALIGNED64_OK) + +#if OPTIMAL_CMP < 64 while (((uintptr_t)out & align_mask) && (len > 0)) { *out++ = *from++; --len; --left; } #endif - if (left < (unsigned)(3 * sizeof(chunk_t))) { + +#ifndef HAVE_MASKED_READWRITE + if (UNLIKELY(left < sizeof(chunk_t))) { while (len > 0) { *out++ = *from++; --len; } + return out; } +#endif + if (len) - return CHUNKMEMSET(out, dist, len); + out = CHUNKMEMSET(out, from, len); return out; } + +static inline uint8_t *CHUNKCOPY_SAFE(uint8_t *out, uint8_t *from, uint64_t len, uint8_t *safe) +{ + if (out == from) + return out + len; + + uint64_t safelen = (safe - out); + len = MIN(len, safelen); + +#ifndef HAVE_MASKED_READWRITE + uint64_t from_dist = (uint64_t)llabs(safe - from); + if (UNLIKELY(from_dist < sizeof(chunk_t) || safelen < sizeof(chunk_t))) { + while (len--) { + *out++ = *from++; + } + + return out; + } +#endif + + return CHUNKMEMSET(out, from, (unsigned)len); +} diff --git a/src/runtime/src/native/external/zlib-ng/cmake/detect-arch.c b/src/runtime/src/native/external/zlib-ng/cmake/detect-arch.c index 92590182c2d..b78b4af2a97 100644 --- a/src/runtime/src/native/external/zlib-ng/cmake/detect-arch.c +++ b/src/runtime/src/native/external/zlib-ng/cmake/detect-arch.c @@ -32,7 +32,7 @@ #endif // PowerPC -#elif defined(__powerpc__) || defined(_ppc__) || defined(__PPC__) +#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) #if defined(__64BIT__) || defined(__powerpc64__) || defined(__ppc64__) #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #error archfound powerpc64le diff --git a/src/runtime/src/native/external/zlib-ng/cmake/detect-intrinsics.cmake b/src/runtime/src/native/external/zlib-ng/cmake/detect-intrinsics.cmake index 14f82fcbf58..b96ac0a4428 100644 --- a/src/runtime/src/native/external/zlib-ng/cmake/detect-intrinsics.cmake +++ b/src/runtime/src/native/external/zlib-ng/cmake/detect-intrinsics.cmake @@ -66,7 +66,7 @@ macro(check_armv6_compiler_flag) return __uqsub16(a, b); #endif } - int main(void) { return 0; }" + int main(void) { return f(1,2); }" HAVE_ARMV6_INTRIN ) set(CMAKE_REQUIRED_FLAGS) @@ -76,14 +76,14 @@ macro(check_avx512_intrinsics) if(NOT NATIVEFLAG) if(CMAKE_C_COMPILER_ID MATCHES "Intel") if(CMAKE_HOST_UNIX OR APPLE) - set(AVX512FLAG "-mavx512f -mavx512dq -mavx512bw -mavx512vl") + set(AVX512FLAG "-mavx512f -mavx512dq -mavx512bw -mavx512vl -mbmi2") else() set(AVX512FLAG "/arch:AVX512") endif() elseif(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_C_COMPILER_ID MATCHES "Clang") # For CPUs that can benefit from AVX512, it seems GCC generates suboptimal # instruction scheduling unless you specify a reasonable -mtune= target - set(AVX512FLAG "-mavx512f -mavx512dq -mavx512bw -mavx512vl") + set(AVX512FLAG "-mavx512f -mavx512dq -mavx512bw -mavx512vl -mbmi2") if(NOT MSVC) check_c_compiler_flag("-mtune=cascadelake" HAVE_CASCADE_LAKE) if(HAVE_CASCADE_LAKE) @@ -114,12 +114,12 @@ macro(check_avx512vnni_intrinsics) if(NOT NATIVEFLAG) if(CMAKE_C_COMPILER_ID MATCHES "Intel") if(CMAKE_HOST_UNIX OR APPLE OR CMAKE_C_COMPILER_ID MATCHES "IntelLLVM") - set(AVX512VNNIFLAG "-mavx512f -mavx512dq -mavx512bw -mavx512vl -mavx512vnni") + set(AVX512VNNIFLAG "-mavx512f -mavx512dq -mavx512bw -mavx512vl -mavx512vnni -mbmi2") else() set(AVX512VNNIFLAG "/arch:AVX512") endif() elseif(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_C_COMPILER_ID MATCHES "Clang") - set(AVX512VNNIFLAG "-mavx512f -mavx512dq -mavx512bw -mavx512vl -mavx512vnni") + set(AVX512VNNIFLAG "-mavx512f -mavx512dq -mavx512bw -mavx512vl -mavx512vnni -mbmi2") if(NOT MSVC) check_c_compiler_flag("-mtune=cascadelake" HAVE_CASCADE_LAKE) if(HAVE_CASCADE_LAKE) @@ -151,12 +151,12 @@ macro(check_avx2_intrinsics) if(NOT NATIVEFLAG) if(CMAKE_C_COMPILER_ID MATCHES "Intel") if(CMAKE_HOST_UNIX OR APPLE) - set(AVX2FLAG "-mavx2") + set(AVX2FLAG "-mavx2 -mbmi2") else() set(AVX2FLAG "/arch:AVX2") endif() elseif(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_C_COMPILER_ID MATCHES "Clang") - set(AVX2FLAG "-mavx2") + set(AVX2FLAG "-mavx2 -mbmi2") elseif(MSVC) set(AVX2FLAG "/arch:AVX2") endif() @@ -433,7 +433,7 @@ macro(check_s390_intrinsics) check_c_source_compiles( "#include #ifndef HWCAP_S390_VXRS - #define HWCAP_S390_VXRS HWCAP_S390_VX + #define HWCAP_S390_VXRS (1 << 11) #endif int main() { return (getauxval(AT_HWCAP) & HWCAP_S390_VXRS); diff --git a/src/runtime/src/native/external/zlib-ng/cmake/detect-sanitizer.cmake b/src/runtime/src/native/external/zlib-ng/cmake/detect-sanitizer.cmake index f9521ec2f54..b71c1a37f37 100644 --- a/src/runtime/src/native/external/zlib-ng/cmake/detect-sanitizer.cmake +++ b/src/runtime/src/native/external/zlib-ng/cmake/detect-sanitizer.cmake @@ -111,6 +111,7 @@ endmacro() macro(add_undefined_sanitizer) set(known_checks + alignment array-bounds bool bounds @@ -137,10 +138,6 @@ macro(add_undefined_sanitizer) vptr ) - # Only check for alignment sanitizer flag if unaligned access is not supported - if(NOT WITH_UNALIGNED) - list(APPEND known_checks alignment) - endif() # Object size sanitizer has no effect at -O0 and produces compiler warning if enabled if(NOT CMAKE_C_FLAGS MATCHES "-O0") list(APPEND known_checks object-size) @@ -153,12 +150,6 @@ macro(add_undefined_sanitizer) add_compile_options(-fsanitize=${supported_checks}) add_link_options(-fsanitize=${supported_checks}) - # Group sanitizer flag -fsanitize=undefined will automatically add alignment, even if - # it is not in our sanitize flag list, so we need to explicitly disable alignment sanitizing. - if(WITH_UNALIGNED) - add_compile_options(-fno-sanitize=alignment) - endif() - add_common_sanitizer_flags() else() message(STATUS "Undefined behavior sanitizer is not supported") diff --git a/src/runtime/src/native/external/zlib-ng/compare256.c b/src/runtime/src/native/external/zlib-ng/compare256.c deleted file mode 100644 index 82551cdd579..00000000000 --- a/src/runtime/src/native/external/zlib-ng/compare256.c +++ /dev/null @@ -1,180 +0,0 @@ -/* compare256.c -- 256 byte memory comparison with match length return - * Copyright (C) 2020 Nathan Moinvaziri - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include "zbuild.h" -#include "zutil_p.h" -#include "fallback_builtins.h" - -/* ALIGNED, byte comparison */ -static inline uint32_t compare256_c_static(const uint8_t *src0, const uint8_t *src1) { - uint32_t len = 0; - - do { - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - if (*src0 != *src1) - return len; - src0 += 1, src1 += 1, len += 1; - } while (len < 256); - - return 256; -} - -Z_INTERNAL uint32_t compare256_c(const uint8_t *src0, const uint8_t *src1) { - return compare256_c_static(src0, src1); -} - -#define LONGEST_MATCH longest_match_c -#define COMPARE256 compare256_c_static - -#include "match_tpl.h" - -#define LONGEST_MATCH_SLOW -#define LONGEST_MATCH longest_match_slow_c -#define COMPARE256 compare256_c_static - -#include "match_tpl.h" - -#if defined(UNALIGNED_OK) && BYTE_ORDER == LITTLE_ENDIAN -/* 16-bit unaligned integer comparison */ -static inline uint32_t compare256_unaligned_16_static(const uint8_t *src0, const uint8_t *src1) { - uint32_t len = 0; - - do { - if (zng_memcmp_2(src0, src1) != 0) - return len + (*src0 == *src1); - src0 += 2, src1 += 2, len += 2; - - if (zng_memcmp_2(src0, src1) != 0) - return len + (*src0 == *src1); - src0 += 2, src1 += 2, len += 2; - - if (zng_memcmp_2(src0, src1) != 0) - return len + (*src0 == *src1); - src0 += 2, src1 += 2, len += 2; - - if (zng_memcmp_2(src0, src1) != 0) - return len + (*src0 == *src1); - src0 += 2, src1 += 2, len += 2; - } while (len < 256); - - return 256; -} - -Z_INTERNAL uint32_t compare256_unaligned_16(const uint8_t *src0, const uint8_t *src1) { - return compare256_unaligned_16_static(src0, src1); -} - -#define LONGEST_MATCH longest_match_unaligned_16 -#define COMPARE256 compare256_unaligned_16_static - -#include "match_tpl.h" - -#define LONGEST_MATCH_SLOW -#define LONGEST_MATCH longest_match_slow_unaligned_16 -#define COMPARE256 compare256_unaligned_16_static - -#include "match_tpl.h" - -#ifdef HAVE_BUILTIN_CTZ -/* 32-bit unaligned integer comparison */ -static inline uint32_t compare256_unaligned_32_static(const uint8_t *src0, const uint8_t *src1) { - uint32_t len = 0; - - do { - uint32_t sv, mv, diff; - - memcpy(&sv, src0, sizeof(sv)); - memcpy(&mv, src1, sizeof(mv)); - - diff = sv ^ mv; - if (diff) { - uint32_t match_byte = __builtin_ctz(diff) / 8; - return len + match_byte; - } - - src0 += 4, src1 += 4, len += 4; - } while (len < 256); - - return 256; -} - -Z_INTERNAL uint32_t compare256_unaligned_32(const uint8_t *src0, const uint8_t *src1) { - return compare256_unaligned_32_static(src0, src1); -} - -#define LONGEST_MATCH longest_match_unaligned_32 -#define COMPARE256 compare256_unaligned_32_static - -#include "match_tpl.h" - -#define LONGEST_MATCH_SLOW -#define LONGEST_MATCH longest_match_slow_unaligned_32 -#define COMPARE256 compare256_unaligned_32_static - -#include "match_tpl.h" - -#endif - -#if defined(UNALIGNED64_OK) && defined(HAVE_BUILTIN_CTZLL) -/* UNALIGNED64_OK, 64-bit integer comparison */ -static inline uint32_t compare256_unaligned_64_static(const uint8_t *src0, const uint8_t *src1) { - uint32_t len = 0; - - do { - uint64_t sv, mv, diff; - - memcpy(&sv, src0, sizeof(sv)); - memcpy(&mv, src1, sizeof(mv)); - - diff = sv ^ mv; - if (diff) { - uint64_t match_byte = __builtin_ctzll(diff) / 8; - return len + (uint32_t)match_byte; - } - - src0 += 8, src1 += 8, len += 8; - } while (len < 256); - - return 256; -} - -Z_INTERNAL uint32_t compare256_unaligned_64(const uint8_t *src0, const uint8_t *src1) { - return compare256_unaligned_64_static(src0, src1); -} - -#define LONGEST_MATCH longest_match_unaligned_64 -#define COMPARE256 compare256_unaligned_64_static - -#include "match_tpl.h" - -#define LONGEST_MATCH_SLOW -#define LONGEST_MATCH longest_match_slow_unaligned_64 -#define COMPARE256 compare256_unaligned_64_static - -#include "match_tpl.h" - -#endif - -#endif diff --git a/src/runtime/src/native/external/zlib-ng/compare256_rle.h b/src/runtime/src/native/external/zlib-ng/compare256_rle.h index 0f3998d4a3f..5edfd734a0d 100644 --- a/src/runtime/src/native/external/zlib-ng/compare256_rle.h +++ b/src/runtime/src/native/external/zlib-ng/compare256_rle.h @@ -4,12 +4,13 @@ */ #include "zbuild.h" +#include "zmemory.h" #include "fallback_builtins.h" typedef uint32_t (*compare256_rle_func)(const uint8_t* src0, const uint8_t* src1); -/* ALIGNED, byte comparison */ -static inline uint32_t compare256_rle_c(const uint8_t *src0, const uint8_t *src1) { +/* 8-bit integer comparison */ +static inline uint32_t compare256_rle_8(const uint8_t *src0, const uint8_t *src1) { uint32_t len = 0; do { @@ -42,29 +43,24 @@ static inline uint32_t compare256_rle_c(const uint8_t *src0, const uint8_t *src1 return 256; } -#ifdef UNALIGNED_OK -/* 16-bit unaligned integer comparison */ -static inline uint32_t compare256_rle_unaligned_16(const uint8_t *src0, const uint8_t *src1) { +/* 16-bit integer comparison */ +static inline uint32_t compare256_rle_16(const uint8_t *src0, const uint8_t *src1) { uint32_t len = 0; - uint16_t src0_cmp, src1_cmp; + uint16_t src0_cmp; - memcpy(&src0_cmp, src0, sizeof(src0_cmp)); + src0_cmp = zng_memread_2(src0); do { - memcpy(&src1_cmp, src1, sizeof(src1_cmp)); - if (src0_cmp != src1_cmp) + if (src0_cmp != zng_memread_2(src1)) return len + (*src0 == *src1); src1 += 2, len += 2; - memcpy(&src1_cmp, src1, sizeof(src1_cmp)); - if (src0_cmp != src1_cmp) + if (src0_cmp != zng_memread_2(src1)) return len + (*src0 == *src1); src1 += 2, len += 2; - memcpy(&src1_cmp, src1, sizeof(src1_cmp)); - if (src0_cmp != src1_cmp) + if (src0_cmp != zng_memread_2(src1)) return len + (*src0 == *src1); src1 += 2, len += 2; - memcpy(&src1_cmp, src1, sizeof(src1_cmp)); - if (src0_cmp != src1_cmp) + if (src0_cmp != zng_memread_2(src1)) return len + (*src0 == *src1); src1 += 2, len += 2; } while (len < 256); @@ -73,22 +69,26 @@ static inline uint32_t compare256_rle_unaligned_16(const uint8_t *src0, const ui } #ifdef HAVE_BUILTIN_CTZ -/* 32-bit unaligned integer comparison */ -static inline uint32_t compare256_rle_unaligned_32(const uint8_t *src0, const uint8_t *src1) { +/* 32-bit integer comparison */ +static inline uint32_t compare256_rle_32(const uint8_t *src0, const uint8_t *src1) { uint32_t sv, len = 0; uint16_t src0_cmp; - memcpy(&src0_cmp, src0, sizeof(src0_cmp)); + src0_cmp = zng_memread_2(src0); sv = ((uint32_t)src0_cmp << 16) | src0_cmp; do { uint32_t mv, diff; - memcpy(&mv, src1, sizeof(mv)); + mv = zng_memread_4(src1); diff = sv ^ mv; if (diff) { +#if BYTE_ORDER == LITTLE_ENDIAN uint32_t match_byte = __builtin_ctz(diff) / 8; +#else + uint32_t match_byte = __builtin_clz(diff) / 8; +#endif return len + match_byte; } @@ -97,28 +97,31 @@ static inline uint32_t compare256_rle_unaligned_32(const uint8_t *src0, const ui return 256; } - #endif -#if defined(UNALIGNED64_OK) && defined(HAVE_BUILTIN_CTZLL) -/* 64-bit unaligned integer comparison */ -static inline uint32_t compare256_rle_unaligned_64(const uint8_t *src0, const uint8_t *src1) { +#ifdef HAVE_BUILTIN_CTZLL +/* 64-bit integer comparison */ +static inline uint32_t compare256_rle_64(const uint8_t *src0, const uint8_t *src1) { uint32_t src0_cmp32, len = 0; uint16_t src0_cmp; uint64_t sv; - memcpy(&src0_cmp, src0, sizeof(src0_cmp)); + src0_cmp = zng_memread_2(src0); src0_cmp32 = ((uint32_t)src0_cmp << 16) | src0_cmp; sv = ((uint64_t)src0_cmp32 << 32) | src0_cmp32; do { uint64_t mv, diff; - memcpy(&mv, src1, sizeof(mv)); + mv = zng_memread_8(src1); diff = sv ^ mv; if (diff) { +#if BYTE_ORDER == LITTLE_ENDIAN uint64_t match_byte = __builtin_ctzll(diff) / 8; +#else + uint64_t match_byte = __builtin_clzll(diff) / 8; +#endif return len + (uint32_t)match_byte; } @@ -127,8 +130,4 @@ static inline uint32_t compare256_rle_unaligned_64(const uint8_t *src0, const ui return 256; } - -#endif - #endif - diff --git a/src/runtime/src/native/external/zlib-ng/configure b/src/runtime/src/native/external/zlib-ng/configure index 8e693fe7b8c..3487d092bc9 100644 --- a/src/runtime/src/native/external/zlib-ng/configure +++ b/src/runtime/src/native/external/zlib-ng/configure @@ -87,7 +87,6 @@ mandir=${mandir-'${prefix}/share/man'} shared_ext='.so' shared=1 gzfileops=1 -unalignedok=1 compat=0 cover=0 build32=0 @@ -106,9 +105,9 @@ floatabi= forcesse2=0 # For CPUs that can benefit from AVX512, it seems GCC generates suboptimal # instruction scheduling unless you specify a reasonable -mtune= target -avx512flag="-mavx512f -mavx512dq -mavx512bw -mavx512vl" +avx512flag="-mavx512f -mavx512dq -mavx512bw -mavx512vl -mbmi2" avx512vnniflag="${avx512flag} -mavx512vnni" -avx2flag="-mavx2" +avx2flag="-mavx2 -mbmi2" sse2flag="-msse2" ssse3flag="-mssse3" sse42flag="-msse4.2" @@ -159,12 +158,11 @@ case "$1" in echo 'usage:' | tee -a configure.log echo ' configure [--prefix=PREFIX] [--eprefix=EXPREFIX]' | tee -a configure.log echo ' [--static] [--32] [--64] [--libdir=LIBDIR] [--sharedlibdir=LIBDIR]' | tee -a configure.log - echo ' [--includedir=INCLUDEDIR] [--archs="-arch i386 -arch x86_64"]' | tee -a configure.log + echo ' [--includedir=INCLUDEDIR] [--mandir=MANDIR] [--archs="-arch i386 -arch x86_64"]' | tee -a configure.log echo ' [--sprefix=SYMBOL_PREFIX] Adds a prefix to all exported symbols' | tee -a configure.log echo ' [--warn] Enables extra compiler warnings' | tee -a configure.log echo ' [--debug] Enables extra debug prints during operation' | tee -a configure.log echo ' [--zlib-compat] Compiles for zlib-compatible API instead of zlib-ng API' | tee -a configure.log - echo ' [--without-unaligned] Compiles without fast unaligned access' | tee -a configure.log echo ' [--without-gzfileops] Compiles without the gzfile parts of the API enabled' | tee -a configure.log echo ' [--without-optimizations] Compiles without support for optional instruction sets' | tee -a configure.log echo ' [--without-new-strategies] Compiles without using new additional deflate strategies' | tee -a configure.log @@ -185,6 +183,7 @@ case "$1" in -l*=* | --libdir=*) libdir=$(echo $1 | sed 's/.*=//'); shift ;; --sharedlibdir=*) sharedlibdir=$(echo $1 | sed 's/.*=//'); shift ;; -i*=* | --includedir=*) includedir=$(echo $1 | sed 's/.*=//');shift ;; + --mandir=*) mandir=$(echo $1 | sed 's/.*=//');shift ;; -u*=* | --uname=*) uname=$(echo $1 | sed 's/.*=//');shift ;; -p* | --prefix) prefix="$2"; shift; shift ;; -e* | --eprefix) exec_prefix="$2"; shift; shift ;; @@ -194,7 +193,6 @@ case "$1" in -s* | --shared | --enable-shared) shared=1; shift ;; -t | --static) shared=0; shift ;; --zlib-compat) compat=1; shift ;; - --without-unaligned) unalignedok=0; shift ;; --without-gzfileops) gzfileops=0; shift ;; --cover) cover=1; shift ;; -3* | --32) build32=1; shift ;; @@ -363,7 +361,7 @@ if test "$gcc" -eq 1 && ($cc $CFLAGS -c $test.c) >> configure.log 2>&1; then uname=$( (uname -s || echo unknown) 2>/dev/null) fi case "$uname" in - Linux* | linux* | GNU | GNU/* | solaris*) + Linux* | linux* | GNU | GNU/* | solaris* | Haiku) LDSHARED=${LDSHARED-"$cc"} LDSHAREDFLAGS="-shared -Wl,-soname,${LIBNAME}.so.${VER1}" ;; *BSD | *bsd* | DragonFly) @@ -875,13 +873,6 @@ else PIC_TESTOBJG="\$(OBJG)" fi -# set architecture alignment requirements -if test $unalignedok -eq 0; then - CFLAGS="${CFLAGS} -DNO_UNALIGNED" - SFLAGS="${SFLAGS} -DNO_UNALIGNED" - echo "Unaligned reads manually disabled." | tee -a configure.log -fi - # enable reduced memory configuration if test $reducedmem -eq 1; then echo "Configuring for reduced memory environment." | tee -a configure.log @@ -1243,7 +1234,7 @@ EOF unsigned int f(unsigned int a, unsigned int b) { return __uqsub16(a, b); } -int main(void) { return 0; } +int main(void) { return f(1, 2); } EOF if try ${CC} ${CFLAGS} ${armv6flag} $test.c; then echo "Checking for ARMv6 intrinsics ... Yes." | tee -a configure.log @@ -1588,8 +1579,8 @@ case "${ARCH}" in if test ${HAVE_AVX512_INTRIN} -eq 1; then CFLAGS="${CFLAGS} -DX86_AVX512" SFLAGS="${SFLAGS} -DX86_AVX512" - ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_avx512.o" - ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_avx512.lo" + ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_avx512.o chunkset_avx512.o" + ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_avx512.lo chunkset_avx512.lo" fi check_mtune_cascadelake_compiler_flag @@ -1976,7 +1967,7 @@ replace_in_file() { # update Makefile with the configure results -INCLUDES="-I$SRCDIR -I$SRCDIR/$ARCHDIR -I$SRCDIR/arch/generic" +INCLUDES="-I$SRCDIR" if [ "$SRCDIR" != "$BUILDDIR" ]; then INCLUDES="-I$BUILDDIR ${INCLUDES}"; fi sed < $SRCDIR/Makefile.in " diff --git a/src/runtime/src/native/external/zlib-ng/crc32_braid.c b/src/runtime/src/native/external/zlib-ng/crc32_braid.c deleted file mode 100644 index 96754b53dff..00000000000 --- a/src/runtime/src/native/external/zlib-ng/crc32_braid.c +++ /dev/null @@ -1,267 +0,0 @@ -/* crc32_braid.c -- compute the CRC-32 of a data stream - * Copyright (C) 1995-2022 Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - * - * This interleaved implementation of a CRC makes use of pipelined multiple - * arithmetic-logic units, commonly found in modern CPU cores. It is due to - * Kadatch and Jenkins (2010). See doc/crc-doc.1.0.pdf in this distribution. - */ - -#include "zbuild.h" -#include "zutil.h" -#include "functable.h" -#include "crc32_braid_p.h" -#include "crc32_braid_tbl.h" - -/* ========================================================================= */ - -const uint32_t * Z_EXPORT PREFIX(get_crc_table)(void) { - return (const uint32_t *)crc_table; -} - -#ifdef ZLIB_COMPAT -unsigned long Z_EXPORT PREFIX(crc32_z)(unsigned long crc, const unsigned char *buf, size_t len) { - if (buf == NULL) return 0; - - return (unsigned long)functable.crc32((uint32_t)crc, buf, len); -} -#else -uint32_t Z_EXPORT PREFIX(crc32_z)(uint32_t crc, const unsigned char *buf, size_t len) { - if (buf == NULL) return 0; - - return functable.crc32(crc, buf, len); -} -#endif - -#ifdef ZLIB_COMPAT -unsigned long Z_EXPORT PREFIX(crc32)(unsigned long crc, const unsigned char *buf, unsigned int len) { - return (unsigned long)PREFIX(crc32_z)((uint32_t)crc, buf, len); -} -#else -uint32_t Z_EXPORT PREFIX(crc32)(uint32_t crc, const unsigned char *buf, uint32_t len) { - return PREFIX(crc32_z)(crc, buf, len); -} -#endif - -/* ========================================================================= */ - -/* - A CRC of a message is computed on N braids of words in the message, where - each word consists of W bytes (4 or 8). If N is 3, for example, then three - running sparse CRCs are calculated respectively on each braid, at these - indices in the array of words: 0, 3, 6, ..., 1, 4, 7, ..., and 2, 5, 8, ... - This is done starting at a word boundary, and continues until as many blocks - of N * W bytes as are available have been processed. The results are combined - into a single CRC at the end. For this code, N must be in the range 1..6 and - W must be 4 or 8. The upper limit on N can be increased if desired by adding - more #if blocks, extending the patterns apparent in the code. In addition, - crc32 tables would need to be regenerated, if the maximum N value is increased. - - N and W are chosen empirically by benchmarking the execution time on a given - processor. The choices for N and W below were based on testing on Intel Kaby - Lake i7, AMD Ryzen 7, ARM Cortex-A57, Sparc64-VII, PowerPC POWER9, and MIPS64 - Octeon II processors. The Intel, AMD, and ARM processors were all fastest - with N=5, W=8. The Sparc, PowerPC, and MIPS64 were all fastest at N=5, W=4. - They were all tested with either gcc or clang, all using the -O3 optimization - level. Your mileage may vary. -*/ - -/* ========================================================================= */ - -#if BYTE_ORDER == LITTLE_ENDIAN -# define ZSWAPWORD(word) (word) -# define BRAID_TABLE crc_braid_table -#elif BYTE_ORDER == BIG_ENDIAN -# if W == 8 -# define ZSWAPWORD(word) ZSWAP64(word) -# elif W == 4 -# define ZSWAPWORD(word) ZSWAP32(word) -# endif -# define BRAID_TABLE crc_braid_big_table -#else -# error "No endian defined" -#endif -#define DO1 c = crc_table[(c ^ *buf++) & 0xff] ^ (c >> 8) -#define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 - -/* ========================================================================= */ -#ifdef W -/* - Return the CRC of the W bytes in the word_t data, taking the - least-significant byte of the word as the first byte of data, without any pre - or post conditioning. This is used to combine the CRCs of each braid. - */ -#if BYTE_ORDER == LITTLE_ENDIAN -static uint32_t crc_word(z_word_t data) { - int k; - for (k = 0; k < W; k++) - data = (data >> 8) ^ crc_table[data & 0xff]; - return (uint32_t)data; -} -#elif BYTE_ORDER == BIG_ENDIAN -static z_word_t crc_word(z_word_t data) { - int k; - for (k = 0; k < W; k++) - data = (data << 8) ^ - crc_big_table[(data >> ((W - 1) << 3)) & 0xff]; - return data; -} -#endif /* BYTE_ORDER */ - -#endif /* W */ - -/* ========================================================================= */ -Z_INTERNAL uint32_t PREFIX(crc32_braid)(uint32_t crc, const uint8_t *buf, size_t len) { - Z_REGISTER uint32_t c; - - /* Pre-condition the CRC */ - c = (~crc) & 0xffffffff; - -#ifdef W - /* If provided enough bytes, do a braided CRC calculation. */ - if (len >= N * W + W - 1) { - size_t blks; - z_word_t const *words; - int k; - - /* Compute the CRC up to a z_word_t boundary. */ - while (len && ((uintptr_t)buf & (W - 1)) != 0) { - len--; - DO1; - } - - /* Compute the CRC on as many N z_word_t blocks as are available. */ - blks = len / (N * W); - len -= blks * N * W; - words = (z_word_t const *)buf; - - z_word_t crc0, word0, comb; -#if N > 1 - z_word_t crc1, word1; -#if N > 2 - z_word_t crc2, word2; -#if N > 3 - z_word_t crc3, word3; -#if N > 4 - z_word_t crc4, word4; -#if N > 5 - z_word_t crc5, word5; -#endif -#endif -#endif -#endif -#endif - /* Initialize the CRC for each braid. */ - crc0 = ZSWAPWORD(c); -#if N > 1 - crc1 = 0; -#if N > 2 - crc2 = 0; -#if N > 3 - crc3 = 0; -#if N > 4 - crc4 = 0; -#if N > 5 - crc5 = 0; -#endif -#endif -#endif -#endif -#endif - /* Process the first blks-1 blocks, computing the CRCs on each braid independently. */ - while (--blks) { - /* Load the word for each braid into registers. */ - word0 = crc0 ^ words[0]; -#if N > 1 - word1 = crc1 ^ words[1]; -#if N > 2 - word2 = crc2 ^ words[2]; -#if N > 3 - word3 = crc3 ^ words[3]; -#if N > 4 - word4 = crc4 ^ words[4]; -#if N > 5 - word5 = crc5 ^ words[5]; -#endif -#endif -#endif -#endif -#endif - words += N; - - /* Compute and update the CRC for each word. The loop should get unrolled. */ - crc0 = BRAID_TABLE[0][word0 & 0xff]; -#if N > 1 - crc1 = BRAID_TABLE[0][word1 & 0xff]; -#if N > 2 - crc2 = BRAID_TABLE[0][word2 & 0xff]; -#if N > 3 - crc3 = BRAID_TABLE[0][word3 & 0xff]; -#if N > 4 - crc4 = BRAID_TABLE[0][word4 & 0xff]; -#if N > 5 - crc5 = BRAID_TABLE[0][word5 & 0xff]; -#endif -#endif -#endif -#endif -#endif - for (k = 1; k < W; k++) { - crc0 ^= BRAID_TABLE[k][(word0 >> (k << 3)) & 0xff]; -#if N > 1 - crc1 ^= BRAID_TABLE[k][(word1 >> (k << 3)) & 0xff]; -#if N > 2 - crc2 ^= BRAID_TABLE[k][(word2 >> (k << 3)) & 0xff]; -#if N > 3 - crc3 ^= BRAID_TABLE[k][(word3 >> (k << 3)) & 0xff]; -#if N > 4 - crc4 ^= BRAID_TABLE[k][(word4 >> (k << 3)) & 0xff]; -#if N > 5 - crc5 ^= BRAID_TABLE[k][(word5 >> (k << 3)) & 0xff]; -#endif -#endif -#endif -#endif -#endif - } - } - - /* Process the last block, combining the CRCs of the N braids at the same time. */ - comb = crc_word(crc0 ^ words[0]); -#if N > 1 - comb = crc_word(crc1 ^ words[1] ^ comb); -#if N > 2 - comb = crc_word(crc2 ^ words[2] ^ comb); -#if N > 3 - comb = crc_word(crc3 ^ words[3] ^ comb); -#if N > 4 - comb = crc_word(crc4 ^ words[4] ^ comb); -#if N > 5 - comb = crc_word(crc5 ^ words[5] ^ comb); -#endif -#endif -#endif -#endif -#endif - words += N; - c = ZSWAPWORD(comb); - - /* Update the pointer to the remaining bytes to process. */ - buf = (const unsigned char *)words; - } - -#endif /* W */ - - /* Complete the computation of the CRC on any remaining bytes. */ - while (len >= 8) { - len -= 8; - DO8; - } - while (len) { - len--; - DO1; - } - - /* Return the CRC, post-conditioned. */ - return c ^ 0xffffffff; -} diff --git a/src/runtime/src/native/external/zlib-ng/crc32_fold.c b/src/runtime/src/native/external/zlib-ng/crc32_fold.c deleted file mode 100644 index 5b3c7c459fd..00000000000 --- a/src/runtime/src/native/external/zlib-ng/crc32_fold.c +++ /dev/null @@ -1,33 +0,0 @@ -/* crc32_fold.c -- crc32 folding interface - * Copyright (C) 2021 Nathan Moinvaziri - * For conditions of distribution and use, see copyright notice in zlib.h - */ -#include "zbuild.h" -#include "functable.h" - -#include "crc32_fold.h" - -#include - -Z_INTERNAL uint32_t crc32_fold_reset_c(crc32_fold *crc) { - crc->value = CRC32_INITIAL_VALUE; - return crc->value; -} - -Z_INTERNAL void crc32_fold_copy_c(crc32_fold *crc, uint8_t *dst, const uint8_t *src, size_t len) { - crc->value = functable.crc32(crc->value, src, len); - memcpy(dst, src, len); -} - -Z_INTERNAL void crc32_fold_c(crc32_fold *crc, const uint8_t *src, size_t len, uint32_t init_crc) { - /* Note: while this is basically the same thing as the vanilla CRC function, we still need - * a functable entry for it so that we can generically dispatch to this function with the - * same arguments for the versions that _do_ do a folding CRC but we don't want a copy. The - * init_crc is an unused argument in this context */ - Z_UNUSED(init_crc); - crc->value = functable.crc32(crc->value, src, len); -} - -Z_INTERNAL uint32_t crc32_fold_final_c(crc32_fold *crc) { - return crc->value; -} diff --git a/src/runtime/src/native/external/zlib-ng/crc32_fold.h b/src/runtime/src/native/external/zlib-ng/crc32_fold.h deleted file mode 100644 index 0d2ff66967d..00000000000 --- a/src/runtime/src/native/external/zlib-ng/crc32_fold.h +++ /dev/null @@ -1,21 +0,0 @@ -/* crc32_fold.h -- crc32 folding interface - * Copyright (C) 2021 Nathan Moinvaziri - * For conditions of distribution and use, see copyright notice in zlib.h - */ -#ifndef CRC32_FOLD_H_ -#define CRC32_FOLD_H_ - -#define CRC32_FOLD_BUFFER_SIZE (16 * 4) -/* sizeof(__m128i) * (4 folds) */ - -typedef struct crc32_fold_s { - uint8_t fold[CRC32_FOLD_BUFFER_SIZE]; - uint32_t value; -} crc32_fold; - -Z_INTERNAL uint32_t crc32_fold_reset_c(crc32_fold *crc); -Z_INTERNAL void crc32_fold_copy_c(crc32_fold *crc, uint8_t *dst, const uint8_t *src, size_t len); -Z_INTERNAL void crc32_fold_c(crc32_fold *crc, const uint8_t *src, size_t len, uint32_t init_crc); -Z_INTERNAL uint32_t crc32_fold_final_c(crc32_fold *crc); - -#endif diff --git a/src/runtime/src/native/external/zlib-ng/deflate.c b/src/runtime/src/native/external/zlib-ng/deflate.c index eb5ae0aabd5..630cce2553b 100644 --- a/src/runtime/src/native/external/zlib-ng/deflate.c +++ b/src/runtime/src/native/external/zlib-ng/deflate.c @@ -239,7 +239,7 @@ Z_INTERNAL deflate_allocs* alloc_deflate(PREFIX3(stream) *strm, int windowBits, int total_size = PAD_64(curr_size + (WINDOW_PAD_SIZE - 1)); /* Allocate buffer, align to 64-byte cacheline, and zerofill the resulting buffer */ - char *original_buf = strm->zalloc(strm->opaque, 1, total_size); + char *original_buf = (char *)strm->zalloc(strm->opaque, 1, total_size); if (original_buf == NULL) return NULL; @@ -248,7 +248,7 @@ Z_INTERNAL deflate_allocs* alloc_deflate(PREFIX3(stream) *strm, int windowBits, /* Initialize alloc_bufs */ deflate_allocs *alloc_bufs = (struct deflate_allocs_s *)(buff + alloc_pos); - alloc_bufs->buf_start = (char *)original_buf; + alloc_bufs->buf_start = original_buf; alloc_bufs->zfree = strm->zfree; /* Assign buffers */ diff --git a/src/runtime/src/native/external/zlib-ng/deflate.h b/src/runtime/src/native/external/zlib-ng/deflate.h index e122ae1aad6..4b79f8f43b2 100644 --- a/src/runtime/src/native/external/zlib-ng/deflate.h +++ b/src/runtime/src/native/external/zlib-ng/deflate.h @@ -12,6 +12,7 @@ #include "zutil.h" #include "zendian.h" +#include "zmemory.h" #include "crc32.h" #ifdef S390_DFLTCC_DEFLATE @@ -243,6 +244,10 @@ struct ALIGNED_(64) internal_state { int nice_match; /* Stop searching when current match exceeds this */ +#if defined(_M_IX86) || defined(_M_ARM) + int padding[2]; +#endif + struct crc32_fold_s ALIGNED_(16) crc_fold; /* used by trees.c: */ @@ -323,7 +328,10 @@ struct ALIGNED_(64) internal_state { /* Number of valid bits in bi_buf. All bits above the last valid bit are always zero. */ /* Reserved for future use and alignment purposes */ - int32_t reserved[11]; + int32_t reserved[19]; +#if defined(_M_IX86) || defined(_M_ARM) + int32_t padding2[4]; +#endif }; typedef enum { @@ -348,7 +356,7 @@ static inline void put_short(deflate_state *s, uint16_t w) { #if BYTE_ORDER == BIG_ENDIAN w = ZSWAP16(w); #endif - memcpy(&s->pending_buf[s->pending], &w, sizeof(w)); + zng_memwrite_2(&s->pending_buf[s->pending], w); s->pending += 2; } @@ -360,7 +368,7 @@ static inline void put_short_msb(deflate_state *s, uint16_t w) { #if BYTE_ORDER == LITTLE_ENDIAN w = ZSWAP16(w); #endif - memcpy(&s->pending_buf[s->pending], &w, sizeof(w)); + zng_memwrite_2(&s->pending_buf[s->pending], w); s->pending += 2; } @@ -372,7 +380,7 @@ static inline void put_uint32(deflate_state *s, uint32_t dw) { #if BYTE_ORDER == BIG_ENDIAN dw = ZSWAP32(dw); #endif - memcpy(&s->pending_buf[s->pending], &dw, sizeof(dw)); + zng_memwrite_4(&s->pending_buf[s->pending], dw); s->pending += 4; } @@ -384,7 +392,7 @@ static inline void put_uint32_msb(deflate_state *s, uint32_t dw) { #if BYTE_ORDER == LITTLE_ENDIAN dw = ZSWAP32(dw); #endif - memcpy(&s->pending_buf[s->pending], &dw, sizeof(dw)); + zng_memwrite_4(&s->pending_buf[s->pending], dw); s->pending += 4; } @@ -396,7 +404,7 @@ static inline void put_uint64(deflate_state *s, uint64_t lld) { #if BYTE_ORDER == BIG_ENDIAN lld = ZSWAP64(lld); #endif - memcpy(&s->pending_buf[s->pending], &lld, sizeof(lld)); + zng_memwrite_8(&s->pending_buf[s->pending], lld); s->pending += 8; } diff --git a/src/runtime/src/native/external/zlib-ng/deflate_quick.c b/src/runtime/src/native/external/zlib-ng/deflate_quick.c index 91c96ac52ed..d5fd986d7ac 100644 --- a/src/runtime/src/native/external/zlib-ng/deflate_quick.c +++ b/src/runtime/src/native/external/zlib-ng/deflate_quick.c @@ -18,7 +18,7 @@ */ #include "zbuild.h" -#include "zutil_p.h" +#include "zmemory.h" #include "deflate.h" #include "deflate_p.h" #include "functable.h" diff --git a/src/runtime/src/native/external/zlib-ng/deflate_rle.c b/src/runtime/src/native/external/zlib-ng/deflate_rle.c index fb17a63c5eb..9e398104834 100644 --- a/src/runtime/src/native/external/zlib-ng/deflate_rle.c +++ b/src/runtime/src/native/external/zlib-ng/deflate_rle.c @@ -5,21 +5,19 @@ */ #include "zbuild.h" -#include "compare256_rle.h" #include "deflate.h" #include "deflate_p.h" #include "functable.h" +#include "compare256_rle.h" -#ifdef UNALIGNED_OK -# if defined(UNALIGNED64_OK) && defined(HAVE_BUILTIN_CTZLL) -# define compare256_rle compare256_rle_unaligned_64 -# elif defined(HAVE_BUILTIN_CTZ) -# define compare256_rle compare256_rle_unaligned_32 -# else -# define compare256_rle compare256_rle_unaligned_16 -# endif +#if OPTIMAL_CMP == 8 +# define compare256_rle compare256_rle_8 +#elif defined(HAVE_BUILTIN_CTZLL) +# define compare256_rle compare256_rle_64 +#elif defined(HAVE_BUILTIN_CTZ) +# define compare256_rle compare256_rle_32 #else -# define compare256_rle compare256_rle_c +# define compare256_rle compare256_rle_16 #endif /* =========================================================================== @@ -59,7 +57,6 @@ Z_INTERNAL block_state deflate_rle(deflate_state *s, int flush) { /* Emit match if have run of STD_MIN_MATCH or longer, else emit literal */ if (match_len >= STD_MIN_MATCH) { Assert(s->strstart <= UINT16_MAX, "strstart should fit in uint16_t"); - Assert(s->match_start <= UINT16_MAX, "match_start should fit in uint16_t"); check_match(s, (Pos)s->strstart, (Pos)(s->strstart - 1), match_len); bflush = zng_tr_tally_dist(s, 1, match_len - STD_MIN_MATCH); diff --git a/src/runtime/src/native/external/zlib-ng/deflate_slow.c b/src/runtime/src/native/external/zlib-ng/deflate_slow.c index b5bea7ab903..4165eea2afe 100644 --- a/src/runtime/src/native/external/zlib-ng/deflate_slow.c +++ b/src/runtime/src/native/external/zlib-ng/deflate_slow.c @@ -79,7 +79,7 @@ Z_INTERNAL block_state deflate_slow(deflate_state *s, int flush) { /* Do not insert strings in hash table beyond this. */ Assert((s->strstart-1) <= UINT16_MAX, "strstart-1 should fit in uint16_t"); - check_match(s, (Pos)(s->strstart-1), s->prev_match, s->prev_length); + check_match(s, (Pos)(s->strstart - 1), s->prev_match, s->prev_length); bflush = zng_tr_tally_dist(s, s->strstart -1 - s->prev_match, s->prev_length - STD_MIN_MATCH); diff --git a/src/runtime/src/native/external/zlib-ng/functable.c b/src/runtime/src/native/external/zlib-ng/functable.c index 495d11edd20..3cee95bff1c 100644 --- a/src/runtime/src/native/external/zlib-ng/functable.c +++ b/src/runtime/src/native/external/zlib-ng/functable.c @@ -5,14 +5,15 @@ #ifndef DISABLE_RUNTIME_CPU_DETECTION #include "zbuild.h" -#include "functable.h" -#include "cpu_features.h" -#include "arch_functions.h" #if defined(_MSC_VER) # include #endif +#include "functable.h" +#include "cpu_features.h" +#include "arch_functions.h" + /* Platform has pointer size atomic store */ #if defined(__GNUC__) || defined(__clang__) # define FUNCTABLE_ASSIGN(VAR, FUNC_NAME) \ @@ -60,9 +61,9 @@ static void init_functable(void) { ft.crc32_fold_reset = &crc32_fold_reset_c; ft.inflate_fast = &inflate_fast_c; ft.slide_hash = &slide_hash_c; - ft.longest_match = &longest_match_generic; - ft.longest_match_slow = &longest_match_slow_generic; - ft.compare256 = &compare256_generic; + ft.longest_match = &longest_match_c; + ft.longest_match_slow = &longest_match_slow_c; + ft.compare256 = &compare256_c; // Select arch-optimized functions @@ -109,7 +110,11 @@ static void init_functable(void) { #endif // X86 - AVX #ifdef X86_AVX2 - if (cf.x86.has_avx2) { + /* BMI2 support is all but implicit with AVX2 but let's sanity check this just in case. Enabling BMI2 allows for + * flagless shifts, resulting in fewer flag stalls for the pipeline, and allows us to set destination registers + * for the shift results as an operand, eliminating several register-register moves when the original value needs + * to remain intact. They also allow for a count operand that isn't the CL register, avoiding contention there */ + if (cf.x86.has_avx2 && cf.x86.has_bmi2) { ft.adler32 = &adler32_avx2; ft.adler32_fold_copy = &adler32_fold_copy_avx2; ft.chunkmemset_safe = &chunkmemset_safe_avx2; @@ -128,6 +133,9 @@ static void init_functable(void) { if (cf.x86.has_avx512_common) { ft.adler32 = &adler32_avx512; ft.adler32_fold_copy = &adler32_fold_copy_avx512; + ft.chunkmemset_safe = &chunkmemset_safe_avx512; + ft.chunksize = &chunksize_avx512; + ft.inflate_fast = &inflate_fast_avx512; } #endif #ifdef X86_AVX512VNNI @@ -272,9 +280,9 @@ static uint32_t adler32_fold_copy_stub(uint32_t adler, uint8_t* dst, const uint8 return functable.adler32_fold_copy(adler, dst, src, len); } -static uint8_t* chunkmemset_safe_stub(uint8_t* out, unsigned dist, unsigned len, unsigned left) { +static uint8_t* chunkmemset_safe_stub(uint8_t* out, uint8_t *from, unsigned len, unsigned left) { init_functable(); - return functable.chunkmemset_safe(out, dist, len, left); + return functable.chunkmemset_safe(out, from, len, left); } static uint32_t chunksize_stub(void) { diff --git a/src/runtime/src/native/external/zlib-ng/functable.h b/src/runtime/src/native/external/zlib-ng/functable.h index 173a030c660..83dda880886 100644 --- a/src/runtime/src/native/external/zlib-ng/functable.h +++ b/src/runtime/src/native/external/zlib-ng/functable.h @@ -27,7 +27,7 @@ struct functable_s { void (* force_init) (void); uint32_t (* adler32) (uint32_t adler, const uint8_t *buf, size_t len); uint32_t (* adler32_fold_copy) (uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len); - uint8_t* (* chunkmemset_safe) (uint8_t *out, unsigned dist, unsigned len, unsigned left); + uint8_t* (* chunkmemset_safe) (uint8_t *out, uint8_t *from, unsigned len, unsigned left); uint32_t (* chunksize) (void); uint32_t (* compare256) (const uint8_t *src0, const uint8_t *src1); uint32_t (* crc32) (uint32_t crc, const uint8_t *buf, size_t len); diff --git a/src/runtime/src/native/external/zlib-ng/infback.c b/src/runtime/src/native/external/zlib-ng/infback.c index 307d05ca3ce..b6d98d4a3fa 100644 --- a/src/runtime/src/native/external/zlib-ng/infback.c +++ b/src/runtime/src/native/external/zlib-ng/infback.c @@ -53,14 +53,19 @@ int32_t ZNG_CONDEXPORT PREFIX(inflateBackInit)(PREFIX3(stream) *strm, int32_t wi Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state *)state; - state->dmax = 32768U; state->wbits = (unsigned int)windowBits; state->wsize = 1U << windowBits; + state->wbufsize = 1U << windowBits; state->window = window; state->wnext = 0; state->whave = 0; - state->sane = 1; state->chunksize = FUNCTABLE_CALL(chunksize)(); +#ifdef INFLATE_STRICT + state->dmax = 32768U; +#endif +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = 1; +#endif return Z_OK; } @@ -97,7 +102,7 @@ int32_t Z_EXPORT PREFIX(inflateBackInit_)(PREFIX3(stream) *strm, int32_t windowB do { \ PULL(); \ have--; \ - hold += ((unsigned)(*next++) << bits); \ + hold += ((uint64_t)(*next++) << bits); \ bits += 8; \ } while (0) @@ -149,7 +154,7 @@ int32_t Z_EXPORT PREFIX(inflateBack)(PREFIX3(stream) *strm, in_func in, void *in z_const unsigned char *next; /* next input */ unsigned char *put; /* next output */ unsigned have, left; /* available input and output */ - uint32_t hold; /* bit buffer */ + uint64_t hold; /* bit buffer */ unsigned bits; /* bits in bit buffer */ unsigned copy; /* number of stored or match bytes to copy */ unsigned char *from; /* where to copy match bytes from */ diff --git a/src/runtime/src/native/external/zlib-ng/inffast_tpl.h b/src/runtime/src/native/external/zlib-ng/inffast_tpl.h index 9ddd187d847..2ec865dbff6 100644 --- a/src/runtime/src/native/external/zlib-ng/inffast_tpl.h +++ b/src/runtime/src/native/external/zlib-ng/inffast_tpl.h @@ -59,13 +59,10 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { unsigned char *beg; /* inflate()'s initial strm->next_out */ unsigned char *end; /* while out < end, enough space available */ unsigned char *safe; /* can use chunkcopy provided out < safe */ -#ifdef INFLATE_STRICT - unsigned dmax; /* maximum distance from zlib header */ -#endif + unsigned char *window; /* allocated sliding window, if wsize != 0 */ unsigned wsize; /* window size or zero if not using window */ unsigned whave; /* valid bytes in the window */ unsigned wnext; /* window write index */ - unsigned char *window; /* allocated sliding window, if wsize != 0 */ /* hold is a local copy of strm->hold. By default, hold satisfies the same invariants that strm->hold does, namely that (hold >> bits) == 0. This @@ -104,18 +101,18 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { with (1<hold >> state->bits) == 0. */ - uint64_t hold; /* local strm->hold */ unsigned bits; /* local strm->bits */ - code const *lcode; /* local strm->lencode */ - code const *dcode; /* local strm->distcode */ + uint64_t hold; /* local strm->hold */ unsigned lmask; /* mask for first level of length codes */ unsigned dmask; /* mask for first level of distance codes */ + code const *lcode; /* local strm->lencode */ + code const *dcode; /* local strm->distcode */ const code *here; /* retrieved table entry */ unsigned op; /* code bits, operation, extra bits, or */ /* window position, window bytes to copy */ unsigned len; /* match length, unused bytes */ - unsigned dist; /* match distance */ unsigned char *from; /* where to copy match from */ + unsigned dist; /* match distance */ unsigned extra_safe; /* copy chunks safely in all cases */ /* copy state to local variables */ @@ -126,9 +123,6 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { beg = out - (start - strm->avail_out); end = out + (strm->avail_out - (INFLATE_FAST_MIN_LEFT - 1)); safe = out + strm->avail_out; -#ifdef INFLATE_STRICT - dmax = state->dmax; -#endif wsize = state->wsize; whave = state->whave; wnext = state->wnext; @@ -143,7 +137,7 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { /* Detect if out and window point to the same memory allocation. In this instance it is necessary to use safe chunk copy functions to prevent overwriting the window. If the window is overwritten then future matches with far distances will fail to copy correctly. */ - extra_safe = (wsize != 0 && out >= window && out + INFLATE_FAST_MIN_LEFT <= window + wsize); + extra_safe = (wsize != 0 && out >= window && out + INFLATE_FAST_MIN_LEFT <= window + state->wbufsize); #define REFILL() do { \ hold |= load_64_bits(in, bits); \ @@ -193,7 +187,7 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { op &= MAX_BITS; /* number of extra bits */ dist += BITS(op); #ifdef INFLATE_STRICT - if (dist > dmax) { + if (dist > state->dmax) { SET_BAD("invalid distance too far back"); break; } @@ -204,11 +198,11 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { if (dist > op) { /* see if copy from window */ op = dist - op; /* distance back in window */ if (op > whave) { +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR if (state->sane) { SET_BAD("invalid distance too far back"); break; } -#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR if (len <= op - whave) { do { *out++ = 0; @@ -226,6 +220,9 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { } while (--len); continue; } +#else + SET_BAD("invalid distance too far back"); + break; #endif } from = window; @@ -238,7 +235,7 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { from += wsize - op; if (op < len) { /* some from end of window */ len -= op; - out = chunkcopy_safe(out, from, op, safe); + out = CHUNKCOPY_SAFE(out, from, op, safe); from = window; /* more from start of window */ op = wnext; /* This (rare) case can create a situation where @@ -248,18 +245,27 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { } if (op < len) { /* still need some from output */ len -= op; - out = chunkcopy_safe(out, from, op, safe); - out = CHUNKUNROLL(out, &dist, &len); - out = chunkcopy_safe(out, out - dist, len, safe); + if (!extra_safe) { + out = CHUNKCOPY_SAFE(out, from, op, safe); + out = CHUNKUNROLL(out, &dist, &len); + out = CHUNKCOPY_SAFE(out, out - dist, len, safe); + } else { + out = chunkcopy_safe(out, from, op, safe); + out = chunkcopy_safe(out, out - dist, len, safe); + } } else { - out = chunkcopy_safe(out, from, len, safe); +#ifndef HAVE_MASKED_READWRITE + if (extra_safe) + out = chunkcopy_safe(out, from, len, safe); + else +#endif + out = CHUNKCOPY_SAFE(out, from, len, safe); } +#ifndef HAVE_MASKED_READWRITE } else if (extra_safe) { /* Whole reference is in range of current output. */ - if (dist >= len || dist >= state->chunksize) out = chunkcopy_safe(out, out - dist, len, safe); - else - out = CHUNKMEMSET_SAFE(out, dist, len, (unsigned)((safe - out) + 1)); +#endif } else { /* Whole reference is in range of current output. No range checks are necessary because we start with room for at least 258 bytes of output, @@ -269,7 +275,7 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) { if (dist >= len || dist >= state->chunksize) out = CHUNKCOPY(out, out - dist, len); else - out = CHUNKMEMSET(out, dist, len); + out = CHUNKMEMSET(out, out - dist, len); } } else if ((op & 64) == 0) { /* 2nd level distance code */ here = dcode + here->val + BITS(op); diff --git a/src/runtime/src/native/external/zlib-ng/inflate.c b/src/runtime/src/native/external/zlib-ng/inflate.c index 956f37db7df..8c373da69d4 100644 --- a/src/runtime/src/native/external/zlib-ng/inflate.c +++ b/src/runtime/src/native/external/zlib-ng/inflate.c @@ -73,13 +73,17 @@ int32_t Z_EXPORT PREFIX(inflateResetKeep)(PREFIX3(stream) *strm) { state->last = 0; state->havedict = 0; state->flags = -1; - state->dmax = 32768U; state->head = NULL; state->hold = 0; state->bits = 0; state->lencode = state->distcode = state->next = state->codes; - state->sane = 1; state->back = -1; +#ifdef INFLATE_STRICT + state->dmax = 32768U; +#endif +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = 1; +#endif INFLATE_RESET_KEEP_HOOK(strm); /* hook for IBM Z DFLTCC */ Tracev((stderr, "inflate: reset\n")); return Z_OK; @@ -170,7 +174,7 @@ Z_INTERNAL inflate_allocs* alloc_inflate(PREFIX3(stream) *strm) { int total_size = PAD_64(curr_size + (WINDOW_PAD_SIZE - 1)); /* Allocate buffer, align to 64-byte cacheline, and zerofill the resulting buffer */ - char *original_buf = strm->zalloc(strm->opaque, 1, total_size); + char *original_buf = (char *)strm->zalloc(strm->opaque, 1, total_size); if (original_buf == NULL) return NULL; @@ -179,7 +183,7 @@ Z_INTERNAL inflate_allocs* alloc_inflate(PREFIX3(stream) *strm) { /* Initialize alloc_bufs */ inflate_allocs *alloc_bufs = (struct inflate_allocs_s *)(buff + alloc_pos); - alloc_bufs->buf_start = (char *)original_buf; + alloc_bufs->buf_start = original_buf; alloc_bufs->zfree = strm->zfree; alloc_bufs->window = (unsigned char *)HINT_ALIGNED_WINDOW((buff + window_pos)); @@ -213,8 +217,8 @@ Z_INTERNAL void free_inflate(PREFIX3(stream) *strm) { * This function is hidden in ZLIB_COMPAT builds. */ int32_t ZNG_CONDEXPORT PREFIX(inflateInit2)(PREFIX3(stream) *strm, int32_t windowBits) { - int32_t ret; struct inflate_state *state; + int32_t ret; /* Initialize functable */ FUNCTABLE_INIT; @@ -236,6 +240,7 @@ int32_t ZNG_CONDEXPORT PREFIX(inflateInit2)(PREFIX3(stream) *strm, int32_t windo state = alloc_bufs->state; state->window = alloc_bufs->window; state->alloc_bufs = alloc_bufs; + state->wbufsize = INFLATE_ADJUST_WINDOW_SIZE((1 << MAX_WBITS) + 64); Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state *)state; @@ -286,7 +291,7 @@ int32_t Z_EXPORT PREFIX(inflatePrime)(PREFIX3(stream) *strm, int32_t bits, int32 if (bits > 16 || state->bits + (unsigned int)bits > 32) return Z_STREAM_ERROR; value &= (1L << bits) - 1; - state->hold += (unsigned)value << state->bits; + state->hold += (uint64_t)value << state->bits; state->bits += (unsigned int)bits; return Z_OK; } @@ -382,7 +387,7 @@ static void updatewindow(PREFIX3(stream) *strm, const uint8_t *end, uint32_t len do { \ if (have == 0) goto inf_leave; \ have--; \ - hold += ((unsigned)(*next++) << bits); \ + hold += ((uint64_t)(*next++) << bits); \ bits += 8; \ } while (0) @@ -472,12 +477,12 @@ int32_t Z_EXPORT PREFIX(inflate)(PREFIX3(stream) *strm, int32_t flush) { struct inflate_state *state; const unsigned char *next; /* next input */ unsigned char *put; /* next output */ + unsigned char *from; /* where to copy match bytes from */ unsigned have, left; /* available input and output */ - uint32_t hold; /* bit buffer */ + uint64_t hold; /* bit buffer */ unsigned bits; /* bits in bit buffer */ uint32_t in, out; /* save starting available input and output */ unsigned copy; /* number of stored or match bytes to copy */ - unsigned char *from; /* where to copy match bytes from */ code here; /* current decoding table entry */ code last; /* parent table entry */ unsigned len; /* length to copy for repeats, bits to drop */ @@ -539,7 +544,9 @@ int32_t Z_EXPORT PREFIX(inflate)(PREFIX3(stream) *strm, int32_t flush) { SET_BAD("invalid window size"); break; } +#ifdef INFLATE_STRICT state->dmax = 1U << len; +#endif state->flags = 0; /* indicate zlib header */ Tracev((stderr, "inflate: zlib header ok\n")); strm->adler = state->check = ADLER32_INITIAL_VALUE; @@ -570,7 +577,7 @@ int32_t Z_EXPORT PREFIX(inflate)(PREFIX3(stream) *strm, int32_t flush) { case TIME: NEEDBITS(32); if (state->head != NULL) - state->head->time = hold; + state->head->time = (unsigned)(hold); if ((state->flags & 0x0200) && (state->wrap & 4)) CRC4(state->check, hold); INITBITS(); @@ -697,7 +704,7 @@ int32_t Z_EXPORT PREFIX(inflate)(PREFIX3(stream) *strm, int32_t flush) { #endif case DICTID: NEEDBITS(32); - strm->adler = state->check = ZSWAP32(hold); + strm->adler = state->check = ZSWAP32((unsigned)hold); INITBITS(); state->mode = DICT; Z_FALLTHROUGH; @@ -1049,11 +1056,11 @@ int32_t Z_EXPORT PREFIX(inflate)(PREFIX3(stream) *strm, int32_t flush) { if (state->offset > copy) { /* copy from window */ copy = state->offset - copy; if (copy > state->whave) { +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR if (state->sane) { SET_BAD("invalid distance too far back"); break; } -#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR Trace((stderr, "inflate.c too far\n")); copy -= state->whave; copy = MIN(copy, state->length); @@ -1065,8 +1072,10 @@ int32_t Z_EXPORT PREFIX(inflate)(PREFIX3(stream) *strm, int32_t flush) { } while (--copy); if (state->length == 0) state->mode = LEN; - break; +#else + SET_BAD("invalid distance too far back"); #endif + break; } if (copy > state->wnext) { copy -= state->wnext; @@ -1081,7 +1090,7 @@ int32_t Z_EXPORT PREFIX(inflate)(PREFIX3(stream) *strm, int32_t flush) { } else { copy = MIN(state->length, left); - put = FUNCTABLE_CALL(chunkmemset_safe)(put, state->offset, copy, left); + put = FUNCTABLE_CALL(chunkmemset_safe)(put, put - state->offset, copy, left); } left -= copy; state->length -= copy; @@ -1119,7 +1128,7 @@ int32_t Z_EXPORT PREFIX(inflate)(PREFIX3(stream) *strm, int32_t flush) { #ifdef GUNZIP state->flags ? hold : #endif - ZSWAP32(hold)) != state->check) { + ZSWAP32((unsigned)hold)) != state->check) { SET_BAD("incorrect data check"); break; } @@ -1297,11 +1306,11 @@ static uint32_t syncsearch(uint32_t *have, const uint8_t *buf, uint32_t len) { } int32_t Z_EXPORT PREFIX(inflateSync)(PREFIX3(stream) *strm) { + struct inflate_state *state; + size_t in, out; /* temporary to save total_in and total_out */ unsigned len; /* number of bytes to look at or looked at */ int flags; /* temporary to save header status */ - size_t in, out; /* temporary to save total_in and total_out */ unsigned char buf[4]; /* to restore bit buffer to byte string */ - struct inflate_state *state; /* check parameters */ if (inflateStateCheck(strm)) @@ -1404,17 +1413,17 @@ int32_t Z_EXPORT PREFIX(inflateCopy)(PREFIX3(stream) *dest, PREFIX3(stream) *sou } int32_t Z_EXPORT PREFIX(inflateUndermine)(PREFIX3(stream) *strm, int32_t subvert) { +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR struct inflate_state *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state *)strm->state; -#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR state->sane = !subvert; return Z_OK; #else + Z_UNUSED(strm); Z_UNUSED(subvert); - state->sane = 1; return Z_DATA_ERROR; #endif } diff --git a/src/runtime/src/native/external/zlib-ng/inflate.h b/src/runtime/src/native/external/zlib-ng/inflate.h index 536da7d1f8f..9d5c482707e 100644 --- a/src/runtime/src/native/external/zlib-ng/inflate.h +++ b/src/runtime/src/native/external/zlib-ng/inflate.h @@ -97,7 +97,7 @@ typedef struct inflate_allocs_s { /* State maintained between inflate() calls -- approximately 7K bytes, not including the allocated sliding window, which is up to 32K bytes. */ struct ALIGNED_(64) inflate_state { - PREFIX3(stream) *strm; /* pointer back to this zlib stream */ + PREFIX3(stream) *strm; /* pointer back to this zlib stream */ inflate_mode mode; /* current inflate mode */ int last; /* true if processing last block */ int wrap; /* bit 0 true for zlib, bit 1 true for gzip, @@ -105,49 +105,64 @@ struct ALIGNED_(64) inflate_state { int havedict; /* true if dictionary provided */ int flags; /* gzip header method and flags, 0 if zlib, or -1 if raw or no header yet */ - unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ + unsigned was; /* initial length of match, for inflateMark */ unsigned long check; /* protected copy of check value */ unsigned long total; /* protected copy of output count */ PREFIX(gz_headerp) head; /* where to save gzip header information */ + int back; /* bits back of last unprocessed length/lit */ + /* sliding window */ unsigned wbits; /* log base 2 of requested window size */ uint32_t wsize; /* window size or zero if not using window */ + uint32_t wbufsize; /* real size of the allocated window buffer, including padding */ uint32_t whave; /* valid bytes in the window */ uint32_t wnext; /* window write index */ unsigned char *window; /* allocated sliding window, if needed */ - - struct crc32_fold_s ALIGNED_(16) crc_fold; + uint32_t chunksize; /* size of memory copying chunk */ /* bit accumulator */ - uint32_t hold; /* input bit accumulator */ + uint64_t hold; /* input bit accumulator */ unsigned bits; /* number of bits in "in" */ + /* fixed and dynamic code tables */ + unsigned lenbits; /* index bits for lencode */ + code const *lencode; /* starting table for length/literal codes */ + code const *distcode; /* starting table for distance codes */ + unsigned distbits; /* index bits for distcode */ /* for string and stored block copying */ uint32_t length; /* literal or length of data to copy */ unsigned offset; /* distance back to copy string from */ /* for table and code decoding */ unsigned extra; /* extra bits needed */ - /* fixed and dynamic code tables */ - code const *lencode; /* starting table for length/literal codes */ - code const *distcode; /* starting table for distance codes */ - unsigned lenbits; /* index bits for lencode */ - unsigned distbits; /* index bits for distcode */ /* dynamic table building */ unsigned ncode; /* number of code length code lengths */ unsigned nlen; /* number of length code lengths */ unsigned ndist; /* number of distance code lengths */ uint32_t have; /* number of code lengths in lens[] */ code *next; /* next available space in codes[] */ + +#if defined(_M_IX86) || defined(_M_ARM) + uint32_t padding[1]; +#endif + struct crc32_fold_s ALIGNED_(16) crc_fold; + uint16_t lens[320]; /* temporary storage for code lengths */ uint16_t work[288]; /* work area for code table building */ code codes[ENOUGH]; /* space for code tables */ - int sane; /* if false, allow invalid distance too far */ - int back; /* bits back of last unprocessed length/lit */ - unsigned was; /* initial length of match */ - uint32_t chunksize; /* size of memory copying chunk */ + inflate_allocs *alloc_bufs; /* struct for handling memory allocations */ + +#ifdef INFLATE_STRICT + unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ +#endif +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + int sane; /* if false, allow invalid distance too far */ +#endif #ifdef HAVE_ARCH_INFLATE_STATE arch_inflate_state arch; /* architecture-specific extensions */ #endif +#if defined(_M_IX86) || defined(_M_ARM) + int padding2[8]; +#endif }; void Z_INTERNAL PREFIX(fixedtables)(struct inflate_state *state); diff --git a/src/runtime/src/native/external/zlib-ng/inflate_p.h b/src/runtime/src/native/external/zlib-ng/inflate_p.h index c324b0486a1..179f76f7340 100644 --- a/src/runtime/src/native/external/zlib-ng/inflate_p.h +++ b/src/runtime/src/native/external/zlib-ng/inflate_p.h @@ -6,6 +6,7 @@ #define INFLATE_P_H #include +#include "zmemory.h" /* Architecture-specific hooks. */ #ifdef S390_DFLTCC_INFLATE @@ -138,8 +139,7 @@ /* Load 64 bits from IN and place the bytes at offset BITS in the result. */ static inline uint64_t load_64_bits(const unsigned char *in, unsigned bits) { - uint64_t chunk; - memcpy(&chunk, in, sizeof(chunk)); + uint64_t chunk = zng_memread_8(in); #if BYTE_ORDER == LITTLE_ENDIAN return chunk << bits; @@ -150,7 +150,7 @@ static inline uint64_t load_64_bits(const unsigned char *in, unsigned bits) { /* Behave like chunkcopy, but avoid writing beyond of legal output. */ static inline uint8_t* chunkcopy_safe(uint8_t *out, uint8_t *from, uint64_t len, uint8_t *safe) { - uint64_t safelen = (safe - out) + 1; + uint64_t safelen = safe - out; len = MIN(len, safelen); int32_t olap_src = from >= out && from < out + len; int32_t olap_dst = out >= from && out < from + len; @@ -174,25 +174,16 @@ static inline uint8_t* chunkcopy_safe(uint8_t *out, uint8_t *from, uint64_t len, * behind or lookahead distance. */ uint64_t non_olap_size = llabs(from - out); // llabs vs labs for compatibility with windows - memcpy(out, from, (size_t)non_olap_size); - out += non_olap_size; - from += non_olap_size; - len -= non_olap_size; - /* So this doesn't give use a worst case scenario of function calls in a loop, - * we want to instead break this down into copy blocks of fixed lengths */ + * we want to instead break this down into copy blocks of fixed lengths + * + * TODO: The memcpy calls aren't inlined on architectures with strict memory alignment + */ while (len) { tocopy = MIN(non_olap_size, len); len -= tocopy; - while (tocopy >= 32) { - memcpy(out, from, 32); - out += 32; - from += 32; - tocopy -= 32; - } - - if (tocopy >= 16) { + while (tocopy >= 16) { memcpy(out, from, 16); out += 16; from += 16; @@ -213,14 +204,7 @@ static inline uint8_t* chunkcopy_safe(uint8_t *out, uint8_t *from, uint64_t len, tocopy -= 4; } - if (tocopy >= 2) { - memcpy(out, from, 2); - out += 2; - from += 2; - tocopy -= 2; - } - - if (tocopy) { + while (tocopy--) { *out++ = *from++; } } diff --git a/src/runtime/src/native/external/zlib-ng/insert_string_tpl.h b/src/runtime/src/native/external/zlib-ng/insert_string_tpl.h index 281c0134631..1548ca741ad 100644 --- a/src/runtime/src/native/external/zlib-ng/insert_string_tpl.h +++ b/src/runtime/src/native/external/zlib-ng/insert_string_tpl.h @@ -22,6 +22,8 @@ * */ +#include "zmemory.h" + #ifndef HASH_CALC_OFFSET # define HASH_CALC_OFFSET 0 #endif @@ -31,13 +33,10 @@ #ifndef HASH_CALC_READ # if BYTE_ORDER == LITTLE_ENDIAN # define HASH_CALC_READ \ - memcpy(&val, strstart, sizeof(val)); + val = zng_memread_4(strstart); # else # define HASH_CALC_READ \ - val = ((uint32_t)(strstart[0])); \ - val |= ((uint32_t)(strstart[1]) << 8); \ - val |= ((uint32_t)(strstart[2]) << 16); \ - val |= ((uint32_t)(strstart[3]) << 24); + val = ZSWAP32(zng_memread_4(strstart)); # endif #endif diff --git a/src/runtime/src/native/external/zlib-ng/match_tpl.h b/src/runtime/src/native/external/zlib-ng/match_tpl.h index 9c258242cd7..47e9aed9f55 100644 --- a/src/runtime/src/native/external/zlib-ng/match_tpl.h +++ b/src/runtime/src/native/external/zlib-ng/match_tpl.h @@ -22,6 +22,9 @@ * IN assertions: cur_match is the head of the hash chain for the current * string (strstart) and its distance is <= MAX_DIST, and prev_length >=1 * OUT assertion: the match length is not greater than s->lookahead + * + * The LONGEST_MATCH_SLOW variant spends more time to attempt to find longer + * matches once a match has already been found. */ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) { unsigned int strstart = s->strstart; @@ -40,10 +43,8 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) { uint32_t chain_length, nice_match, best_len, offset; uint32_t lookahead = s->lookahead; Pos match_offset = 0; -#ifdef UNALIGNED_OK - uint8_t scan_start[8]; -#endif - uint8_t scan_end[8]; + uint64_t scan_start; + uint64_t scan_end; #define GOTO_NEXT_CHAIN \ if (--chain_length && (cur_match = prev[cur_match & wmask]) > limit) \ @@ -59,26 +60,14 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) { * to find the next best match length. */ offset = best_len-1; -#ifdef UNALIGNED_OK if (best_len >= sizeof(uint32_t)) { offset -= 2; -#ifdef UNALIGNED64_OK if (best_len >= sizeof(uint64_t)) offset -= 4; -#endif } -#endif -#ifdef UNALIGNED64_OK - memcpy(scan_start, scan, sizeof(uint64_t)); - memcpy(scan_end, scan+offset, sizeof(uint64_t)); -#elif defined(UNALIGNED_OK) - memcpy(scan_start, scan, sizeof(uint32_t)); - memcpy(scan_end, scan+offset, sizeof(uint32_t)); -#else - scan_end[0] = *(scan+offset); - scan_end[1] = *(scan+offset+1); -#endif + scan_start = zng_memread_8(scan); + scan_end = zng_memread_8(scan+offset); mbase_end = (mbase_start+offset); /* Do not waste too much time if we already have a good match */ @@ -138,39 +127,28 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) { * that depend on those values. However the length of the match is limited to the * lookahead, so the output of deflate is not affected by the uninitialized values. */ -#ifdef UNALIGNED_OK if (best_len < sizeof(uint32_t)) { for (;;) { - if (zng_memcmp_2(mbase_end+cur_match, scan_end) == 0 && - zng_memcmp_2(mbase_start+cur_match, scan_start) == 0) + if (zng_memcmp_2(mbase_end+cur_match, &scan_end) == 0 && + zng_memcmp_2(mbase_start+cur_match, &scan_start) == 0) break; GOTO_NEXT_CHAIN; } -# ifdef UNALIGNED64_OK } else if (best_len >= sizeof(uint64_t)) { for (;;) { - if (zng_memcmp_8(mbase_end+cur_match, scan_end) == 0 && - zng_memcmp_8(mbase_start+cur_match, scan_start) == 0) + if (zng_memcmp_8(mbase_end+cur_match, &scan_end) == 0 && + zng_memcmp_8(mbase_start+cur_match, &scan_start) == 0) break; GOTO_NEXT_CHAIN; } -# endif } else { for (;;) { - if (zng_memcmp_4(mbase_end+cur_match, scan_end) == 0 && - zng_memcmp_4(mbase_start+cur_match, scan_start) == 0) + if (zng_memcmp_4(mbase_end+cur_match, &scan_end) == 0 && + zng_memcmp_4(mbase_start+cur_match, &scan_start) == 0) break; GOTO_NEXT_CHAIN; } } -#else - for (;;) { - if (mbase_end[cur_match] == scan_end[0] && mbase_end[cur_match+1] == scan_end[1] && - mbase_start[cur_match] == scan[0] && mbase_start[cur_match+1] == scan[1]) - break; - GOTO_NEXT_CHAIN; - } -#endif uint32_t len = COMPARE256(scan+2, mbase_start+cur_match+2) + 2; Assert(scan+len <= window+(unsigned)(s->window_size-1), "wild scan"); @@ -186,24 +164,13 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) { return best_len; offset = best_len-1; -#ifdef UNALIGNED_OK if (best_len >= sizeof(uint32_t)) { offset -= 2; -#ifdef UNALIGNED64_OK if (best_len >= sizeof(uint64_t)) offset -= 4; -#endif } -#endif -#ifdef UNALIGNED64_OK - memcpy(scan_end, scan+offset, sizeof(uint64_t)); -#elif defined(UNALIGNED_OK) - memcpy(scan_end, scan+offset, sizeof(uint32_t)); -#else - scan_end[0] = *(scan+offset); - scan_end[1] = *(scan+offset+1); -#endif + scan_end = zng_memread_8(scan+offset); #ifdef LONGEST_MATCH_SLOW /* Look for a better string offset */ @@ -281,4 +248,3 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) { #undef LONGEST_MATCH_SLOW #undef LONGEST_MATCH -#undef COMPARE256 diff --git a/src/runtime/src/native/external/zlib-ng/slide_hash.c b/src/runtime/src/native/external/zlib-ng/slide_hash.c deleted file mode 100644 index b9fbbdb69f8..00000000000 --- a/src/runtime/src/native/external/zlib-ng/slide_hash.c +++ /dev/null @@ -1,52 +0,0 @@ -/* slide_hash.c -- slide hash table C implementation - * - * Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include "zbuild.h" -#include "deflate.h" - -/* =========================================================================== - * Slide the hash table when sliding the window down (could be avoided with 32 - * bit values at the expense of memory usage). We slide even when level == 0 to - * keep the hash table consistent if we switch back to level > 0 later. - */ -static inline void slide_hash_c_chain(Pos *table, uint32_t entries, uint16_t wsize) { -#ifdef NOT_TWEAK_COMPILER - table += entries; - do { - unsigned m; - m = *--table; - *table = (Pos)(m >= wsize ? m-wsize : 0); - /* If entries is not on any hash chain, prev[entries] is garbage but - * its value will never be used. - */ - } while (--entries); -#else - { - /* As of I make this change, gcc (4.8.*) isn't able to vectorize - * this hot loop using saturated-subtraction on x86-64 architecture. - * To avoid this defect, we can change the loop such that - * o. the pointer advance forward, and - * o. demote the variable 'm' to be local to the loop, and - * choose type "Pos" (instead of 'unsigned int') for the - * variable to avoid unnecessary zero-extension. - */ - unsigned int i; - Pos *q = table; - for (i = 0; i < entries; i++) { - Pos m = *q; - Pos t = (Pos)wsize; - *q++ = (Pos)(m >= t ? m-t: 0); - } - } -#endif /* NOT_TWEAK_COMPILER */ -} - -Z_INTERNAL void slide_hash_c(deflate_state *s) { - uint16_t wsize = (uint16_t)s->w_size; - - slide_hash_c_chain(s->head, HASH_SIZE, wsize); - slide_hash_c_chain(s->prev, wsize, wsize); -} diff --git a/src/runtime/src/native/external/zlib-ng/trees_emit.h b/src/runtime/src/native/external/zlib-ng/trees_emit.h index 922daae509f..b5295ee4836 100644 --- a/src/runtime/src/native/external/zlib-ng/trees_emit.h +++ b/src/runtime/src/native/external/zlib-ng/trees_emit.h @@ -38,6 +38,10 @@ extern Z_INTERNAL const int base_dist[D_CODES]; /* If not enough room in bi_buf, use (valid) bits from bi_buf and * (64 - bi_valid) bits from value, leaving (width - (64-bi_valid)) * unused bits in value. + * + * NOTE: Static analyzers can't evaluate value of total_bits, so we + * also need to make sure bi_valid is within acceptable range, + * otherwise the shifts will overflow. */ #define send_bits(s, t_val, t_len, bi_buf, bi_valid) {\ uint64_t val = (uint64_t)t_val;\ @@ -45,10 +49,10 @@ extern Z_INTERNAL const int base_dist[D_CODES]; uint32_t total_bits = bi_valid + len;\ send_bits_trace(s, val, len);\ sent_bits_add(s, len);\ - if (total_bits < BIT_BUF_SIZE) {\ + if (total_bits < BIT_BUF_SIZE && bi_valid < BIT_BUF_SIZE) {\ bi_buf |= val << bi_valid;\ bi_valid = total_bits;\ - } else if (bi_valid == BIT_BUF_SIZE) {\ + } else if (bi_valid >= BIT_BUF_SIZE) {\ put_uint64(s, bi_buf);\ bi_buf = val;\ bi_valid = len;\ diff --git a/src/runtime/src/native/external/zlib-ng/win32/Makefile.a64 b/src/runtime/src/native/external/zlib-ng/win32/Makefile.a64 index 9f8d6fb7fac..3209f6a3051 100644 --- a/src/runtime/src/native/external/zlib-ng/win32/Makefile.a64 +++ b/src/runtime/src/native/external/zlib-ng/win32/Makefile.a64 @@ -183,7 +183,7 @@ adler32.obj: $(TOP)/adler32.c $(TOP)/zbuild.h $(TOP)/functable.h $(TOP)/adler32_ adler32_c.obj: $(TOP)/arch/generic/adler32_c.c $(TOP)/zbuild.h $(TOP)/functable.h $(TOP)/adler32_p.h adler32_fold_c.obj: $(TOP)/arch/generic/adler32_fold_c.c $(TOP)/zbuild.h $(TOP)/functable.h chunkset_c.obj: $(TOP)/arch/generic/chunkset_c.c $(TOP)/zbuild.h $(TOP)/chunkset_tpl.h $(TOP)/inffast_tpl.h -compare256_c.obj: $(TOP)/arch/generic/compare256_c.c $(TOP)/zbuild.h $(TOP)/zutil_p.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h +compare256_c.obj: $(TOP)/arch/generic/compare256_c.c $(TOP)/zbuild.h $(TOP)/zmemory.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h compress.obj: $(TOP)/compress.c $(TOP)/zbuild.h $(TOP)/zutil.h cpu_features.obj: $(TOP)/cpu_features.c $(TOP)/cpu_features.h $(TOP)/zbuild.h crc32.obj: $(TOP)/crc32.c $(TOP)/zbuild.h $(TOP)/functable.h $(TOP)/crc32_braid_tbl.h @@ -194,7 +194,7 @@ deflate.obj: $(TOP)/deflate.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p. deflate_fast.obj: $(TOP)/deflate_fast.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h deflate_huff.obj: $(TOP)/deflate_huff.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h deflate_medium.obj: $(TOP)/deflate_medium.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h -deflate_quick.obj: $(TOP)/deflate_quick.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h $(TOP)/trees_emit.h $(TOP)/zutil_p.h +deflate_quick.obj: $(TOP)/deflate_quick.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h $(TOP)/trees_emit.h $(TOP)/zmemory.h deflate_rle.obj: $(TOP)/deflate_rle.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h $(TOP)/compare256_rle.h deflate_slow.obj: $(TOP)/deflate_slow.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h deflate_stored.obj: $(TOP)/deflate_stored.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h diff --git a/src/runtime/src/native/external/zlib-ng/win32/Makefile.arm b/src/runtime/src/native/external/zlib-ng/win32/Makefile.arm index cab999dfe03..54da045ffdd 100644 --- a/src/runtime/src/native/external/zlib-ng/win32/Makefile.arm +++ b/src/runtime/src/native/external/zlib-ng/win32/Makefile.arm @@ -204,7 +204,7 @@ adler32.obj: $(TOP)/adler32.c $(TOP)/zbuild.h $(TOP)/functable.h $(TOP)/adler32_ adler32_c.obj: $(TOP)/arch/generic/adler32_c.c $(TOP)/zbuild.h $(TOP)/functable.h $(TOP)/adler32_p.h adler32_fold_c.obj: $(TOP)/arch/generic/adler32_fold_c.c $(TOP)/zbuild.h $(TOP)/functable.h chunkset_c.obj: $(TOP)/arch/generic/chunkset_c.c $(TOP)/zbuild.h $(TOP)/chunkset_tpl.h $(TOP)/inffast_tpl.h -compare256_c.obj: $(TOP)/arch/generic/compare256_c.c $(TOP)/zbuild.h $(TOP)/zutil_p.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h +compare256_c.obj: $(TOP)/arch/generic/compare256_c.c $(TOP)/zbuild.h $(TOP)/zmemory.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h compress.obj: $(TOP)/compress.c $(TOP)/zbuild.h $(TOP)/zutil.h cpu_features.obj: $(TOP)/cpu_features.c $(TOP)/cpu_features.h $(TOP)/zbuild.h crc32.obj: $(TOP)/crc32.c $(TOP)/zbuild.h $(TOP)/functable.h $(TOP)/crc32_braid_tbl.h @@ -215,7 +215,7 @@ deflate.obj: $(TOP)/deflate.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p. deflate_fast.obj: $(TOP)/deflate_fast.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h deflate_huff.obj: $(TOP)/deflate_huff.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h deflate_medium.obj: $(TOP)/deflate_medium.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h -deflate_quick.obj: $(TOP)/deflate_quick.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h $(TOP)/trees_emit.h $(TOP)/zutil_p.h +deflate_quick.obj: $(TOP)/deflate_quick.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h $(TOP)/trees_emit.h $(TOP)/zmemory.h deflate_rle.obj: $(TOP)/deflate_rle.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h $(TOP)/compare256_rle.h deflate_slow.obj: $(TOP)/deflate_slow.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h deflate_stored.obj: $(TOP)/deflate_stored.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h diff --git a/src/runtime/src/native/external/zlib-ng/win32/Makefile.msc b/src/runtime/src/native/external/zlib-ng/win32/Makefile.msc index 8392fe46e7e..62ca621aef7 100644 --- a/src/runtime/src/native/external/zlib-ng/win32/Makefile.msc +++ b/src/runtime/src/native/external/zlib-ng/win32/Makefile.msc @@ -212,9 +212,9 @@ chunkset_c.obj: $(TOP)/arch/generic/chunkset_c.c $(TOP)/zbuild.h $(TOP)/chunkset chunkset_avx2.obj: $(TOP)/arch/x86/chunkset_avx2.c $(TOP)/zbuild.h $(TOP)/chunkset_tpl.h $(TOP)/inffast_tpl.h $(TOP)/arch/generic/chunk_permute_table.h chunkset_sse2.obj: $(TOP)/arch/x86/chunkset_sse2.c $(TOP)/zbuild.h $(TOP)/chunkset_tpl.h $(TOP)/inffast_tpl.h chunkset_ssse3.obj: $(TOP)/arch/x86/chunkset_ssse3.c $(TOP)/zbuild.h $(TOP)/chunkset_tpl.h $(TOP)/inffast_tpl.h $(TOP)/arch/generic/chunk_permute_table.h -compare256_c.obj: $(TOP)/arch/generic/compare256_c.c $(TOP)/zbuild.h $(TOP)/zutil_p.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h -compare256_avx2.obj: $(TOP)/arch/x86/compare256_avx2.c $(TOP)/zbuild.h $(TOP)/zutil_p.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h -compare256_sse2.obj: $(TOP)/arch/x86/compare256_sse2.c $(TOP)/zbuild.h $(TOP)/zutil_p.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h +compare256_c.obj: $(TOP)/arch/generic/compare256_c.c $(TOP)/zbuild.h $(TOP)/zmemory.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h +compare256_avx2.obj: $(TOP)/arch/x86/compare256_avx2.c $(TOP)/zbuild.h $(TOP)/zmemory.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h +compare256_sse2.obj: $(TOP)/arch/x86/compare256_sse2.c $(TOP)/zbuild.h $(TOP)/zmemory.h $(TOP)/deflate.h $(TOP)/fallback_builtins.h $(TOP)/match_tpl.h compress.obj: $(TOP)/compress.c $(TOP)/zbuild.h $(TOP)/zutil.h cpu_features.obj: $(TOP)/cpu_features.c $(TOP)/cpu_features.h $(TOP)/zbuild.h crc32.obj: $(TOP)/crc32.c $(TOP)/zbuild.h $(TOP)/functable.h $(TOP)/crc32_braid_tbl.h @@ -226,7 +226,7 @@ deflate.obj: $(TOP)/deflate.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p. deflate_fast.obj: $(TOP)/deflate_fast.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h deflate_huff.obj: $(TOP)/deflate_huff.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h deflate_medium.obj: $(TOP)/deflate_medium.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h -deflate_quick.obj: $(TOP)/deflate_quick.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h $(TOP)/trees_emit.h $(TOP)/zutil_p.h +deflate_quick.obj: $(TOP)/deflate_quick.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h $(TOP)/trees_emit.h $(TOP)/zmemory.h deflate_rle.obj: $(TOP)/deflate_rle.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h $(TOP)/compare256_rle.h deflate_slow.obj: $(TOP)/deflate_slow.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h deflate_stored.obj: $(TOP)/deflate_stored.c $(TOP)/zbuild.h $(TOP)/deflate.h $(TOP)/deflate_p.h $(TOP)/functable.h diff --git a/src/runtime/src/native/external/zlib-ng/zbuild.h b/src/runtime/src/native/external/zlib-ng/zbuild.h index 9157eef9e35..157ab6ffedc 100644 --- a/src/runtime/src/native/external/zlib-ng/zbuild.h +++ b/src/runtime/src/native/external/zlib-ng/zbuild.h @@ -200,6 +200,9 @@ # define ALIGNED_(x) __attribute__ ((aligned(x))) #elif defined(_MSC_VER) # define ALIGNED_(x) __declspec(align(x)) +#else +/* TODO: Define ALIGNED_ for your compiler */ +# define ALIGNED_(x) #endif #ifdef HAVE_BUILTIN_ASSUME_ALIGNED @@ -225,7 +228,7 @@ # include extern int Z_INTERNAL z_verbose; extern void Z_INTERNAL z_error(const char *m); -# define Assert(cond, msg) {if (!(cond)) z_error(msg);} +# define Assert(cond, msg) {int _cond = (cond); if (!_cond) z_error(msg);} # define Trace(x) {if (z_verbose >= 0) fprintf x;} # define Tracev(x) {if (z_verbose > 0) fprintf x;} # define Tracevv(x) {if (z_verbose > 1) fprintf x;} @@ -240,29 +243,42 @@ # define Tracecv(c, x) #endif -#ifndef NO_UNALIGNED +/* OPTIMAL_CMP values determine the comparison width: + * 64: Best for 64-bit architectures with unaligned access + * 32: Best for 32-bit architectures with unaligned access + * 16: Safe default for unknown architectures + * 8: Safe fallback for architectures without unaligned access + * Note: The unaligned access mentioned is cpu-support, this allows compiler or + * separate unaligned intrinsics to utilize safe unaligned access, without + * utilizing unaligned C pointers that are known to have undefined behavior. + */ +#if !defined(OPTIMAL_CMP) # if defined(__x86_64__) || defined(_M_X64) || defined(__amd64__) || defined(_M_AMD64) -# define UNALIGNED_OK -# define UNALIGNED64_OK +# define OPTIMAL_CMP 64 # elif defined(__i386__) || defined(__i486__) || defined(__i586__) || \ defined(__i686__) || defined(_X86_) || defined(_M_IX86) -# define UNALIGNED_OK +# define OPTIMAL_CMP 32 # elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) -# if (defined(__GNUC__) && defined(__ARM_FEATURE_UNALIGNED)) || !defined(__GNUC__) -# define UNALIGNED_OK -# define UNALIGNED64_OK +# if defined(__ARM_FEATURE_UNALIGNED) || defined(_WIN32) +# define OPTIMAL_CMP 64 +# else +# define OPTIMAL_CMP 8 # endif -# elif defined(__arm__) || (_M_ARM >= 7) -# if (defined(__GNUC__) && defined(__ARM_FEATURE_UNALIGNED)) || !defined(__GNUC__) -# define UNALIGNED_OK +# elif defined(__arm__) || defined(_M_ARM) +# if defined(__ARM_FEATURE_UNALIGNED) || defined(_WIN32) +# define OPTIMAL_CMP 32 +# else +# define OPTIMAL_CMP 8 # endif # elif defined(__powerpc64__) || defined(__ppc64__) -# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -# define UNALIGNED_OK -# define UNALIGNED64_OK -# endif +# define OPTIMAL_CMP 64 +# elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) +# define OPTIMAL_CMP 32 # endif #endif +#if !defined(OPTIMAL_CMP) +# define OPTIMAL_CMP 16 +#endif #if defined(__has_feature) # if __has_feature(address_sanitizer) @@ -275,8 +291,8 @@ /* * __asan_loadN() and __asan_storeN() calls are inserted by compilers in order to check memory accesses. * They can be called manually too, with the following caveats: - * gcc says: "warning: implicit declaration of function ‘...’" - * g++ says: "error: new declaration ‘...’ ambiguates built-in declaration ‘...’" + * gcc says: "warning: implicit declaration of function '...'" + * g++ says: "error: new declaration '...' ambiguates built-in declaration '...'" * Accommodate both. */ #ifdef Z_ADDRESS_SANITIZER diff --git a/src/runtime/src/native/external/zlib-ng/zlib-ng.h.in b/src/runtime/src/native/external/zlib-ng/zlib-ng.h.in index 7f7f03ee155..93c7c86f26d 100644 --- a/src/runtime/src/native/external/zlib-ng/zlib-ng.h.in +++ b/src/runtime/src/native/external/zlib-ng/zlib-ng.h.in @@ -48,11 +48,11 @@ extern "C" { #endif -#define ZLIBNG_VERSION "2.2.1" -#define ZLIBNG_VERNUM 0x020201F0L /* MMNNRRSM: major minor revision status modified */ +#define ZLIBNG_VERSION "2.2.4" +#define ZLIBNG_VERNUM 0x020204F0L /* MMNNRRSM: major minor revision status modified */ #define ZLIBNG_VER_MAJOR 2 #define ZLIBNG_VER_MINOR 2 -#define ZLIBNG_VER_REVISION 1 +#define ZLIBNG_VER_REVISION 4 #define ZLIBNG_VER_STATUS F /* 0=devel, 1-E=beta, F=Release (DEPRECATED) */ #define ZLIBNG_VER_STATUSH 0xF /* Hex values: 0=devel, 1-E=beta, F=Release */ #define ZLIBNG_VER_MODIFIED 0 /* non-zero if modified externally from zlib-ng */ @@ -880,7 +880,7 @@ int32_t zng_inflateSetDictionary(zng_stream *strm, const uint8_t *dictionary, ui deflateSetDictionary). For raw inflate, this function can be called at any time to set the dictionary. If the provided dictionary is smaller than the window and there is already data in the window, then the provided dictionary - will amend what's there. The application must insure that the dictionary + will amend what's there. The application must ensure that the dictionary that was used for compression is provided. inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a diff --git a/src/runtime/src/native/external/zlib-ng/zlib.h.in b/src/runtime/src/native/external/zlib-ng/zlib.h.in index 3dceaa3344b..b8fa106ef16 100644 --- a/src/runtime/src/native/external/zlib-ng/zlib.h.in +++ b/src/runtime/src/native/external/zlib-ng/zlib.h.in @@ -49,11 +49,11 @@ extern "C" { #endif -#define ZLIBNG_VERSION "2.2.1" -#define ZLIBNG_VERNUM 0x020201F0L /* MMNNRRSM: major minor revision status modified */ +#define ZLIBNG_VERSION "2.2.4" +#define ZLIBNG_VERNUM 0x020204F0L /* MMNNRRSM: major minor revision status modified */ #define ZLIBNG_VER_MAJOR 2 #define ZLIBNG_VER_MINOR 2 -#define ZLIBNG_VER_REVISION 1 +#define ZLIBNG_VER_REVISION 4 #define ZLIBNG_VER_STATUS F /* 0=devel, 1-E=beta, F=Release (DEPRECATED) */ #define ZLIBNG_VER_STATUSH 0xF /* Hex values: 0=devel, 1-E=beta, F=Release */ #define ZLIBNG_VER_MODIFIED 0 /* non-zero if modified externally from zlib-ng */ @@ -890,7 +890,7 @@ Z_EXTERN int Z_EXPORT inflateSetDictionary(z_stream *strm, const unsigned char * deflateSetDictionary). For raw inflate, this function can be called at any time to set the dictionary. If the provided dictionary is smaller than the window and there is already data in the window, then the provided dictionary - will amend what's there. The application must insure that the dictionary + will amend what's there. The application must ensure that the dictionary that was used for compression is provided. inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a diff --git a/src/runtime/src/native/external/zlib-ng/zmemory.h b/src/runtime/src/native/external/zlib-ng/zmemory.h new file mode 100644 index 00000000000..fc850a72273 --- /dev/null +++ b/src/runtime/src/native/external/zlib-ng/zmemory.h @@ -0,0 +1,99 @@ +/* zmemory.h -- Private inline functions used internally in zlib-ng + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#ifndef _ZMEMORY_H +#define _ZMEMORY_H + +#if defined(__GNUC__) && (__GNUC__ >= 4) +# define HAVE_MAY_ALIAS +#endif + +static inline uint16_t zng_memread_2(const void *ptr) { +#if defined(HAVE_MAY_ALIAS) + typedef struct { uint16_t val; } __attribute__ ((__packed__, __may_alias__)) unaligned_uint16_t; + return ((const unaligned_uint16_t *)ptr)->val; +#else + uint16_t val; + memcpy(&val, ptr, sizeof(val)); + return val; +#endif +} + +static inline uint32_t zng_memread_4(const void *ptr) { +#if defined(HAVE_MAY_ALIAS) + typedef struct { uint32_t val; } __attribute__ ((__packed__, __may_alias__)) unaligned_uint32_t; + return ((const unaligned_uint32_t *)ptr)->val; +#else + uint32_t val; + memcpy(&val, ptr, sizeof(val)); + return val; +#endif +} + +static inline uint64_t zng_memread_8(const void *ptr) { +#if defined(HAVE_MAY_ALIAS) + typedef struct { uint64_t val; } __attribute__ ((__packed__, __may_alias__)) unaligned_uint64_t; + return ((const unaligned_uint64_t *)ptr)->val; +#else + uint64_t val; + memcpy(&val, ptr, sizeof(val)); + return val; +#endif +} + +static inline void zng_memwrite_2(void *ptr, uint16_t val) { +#if defined(HAVE_MAY_ALIAS) + typedef struct { uint16_t val; } __attribute__ ((__packed__, __may_alias__)) unaligned_uint16_t; + ((unaligned_uint16_t *)ptr)->val = val; +#else + memcpy(ptr, &val, sizeof(val)); +#endif +} + +static inline void zng_memwrite_4(void *ptr, uint32_t val) { +#if defined(HAVE_MAY_ALIAS) + typedef struct { uint32_t val; } __attribute__ ((__packed__, __may_alias__)) unaligned_uint32_t; + ((unaligned_uint32_t *)ptr)->val = val; +#else + memcpy(ptr, &val, sizeof(val)); +#endif +} + +static inline void zng_memwrite_8(void *ptr, uint64_t val) { +#if defined(HAVE_MAY_ALIAS) + typedef struct { uint64_t val; } __attribute__ ((__packed__, __may_alias__)) unaligned_uint64_t; + ((unaligned_uint64_t *)ptr)->val = val; +#else + memcpy(ptr, &val, sizeof(val)); +#endif +} + +/* Use zng_memread_* instead of memcmp to avoid older compilers not converting memcmp + calls to unaligned comparisons when unaligned access is supported. Use memcmp only when + unaligned support is not available to avoid an extra call to memcpy. */ +static inline int32_t zng_memcmp_2(const void *src0, const void *src1) { +#if defined(HAVE_MAY_ALIAS) + return zng_memread_2(src0) != zng_memread_2(src1); +#else + return memcmp(src0, src1, 2); +#endif +} + +static inline int32_t zng_memcmp_4(const void *src0, const void *src1) { +#if defined(HAVE_MAY_ALIAS) + return zng_memread_4(src0) != zng_memread_4(src1); +#else + return memcmp(src0, src1, 4); +#endif +} + +static inline int32_t zng_memcmp_8(const void *src0, const void *src1) { +#if defined(HAVE_MAY_ALIAS) + return zng_memread_8(src0) != zng_memread_8(src1); +#else + return memcmp(src0, src1, 8); +#endif +} + +#endif diff --git a/src/runtime/src/native/external/zlib-ng/zutil.c b/src/runtime/src/native/external/zlib-ng/zutil.c index 39fbceb4a01..99818c3a7b9 100644 --- a/src/runtime/src/native/external/zlib-ng/zutil.c +++ b/src/runtime/src/native/external/zlib-ng/zutil.c @@ -21,7 +21,7 @@ z_const char * const PREFIX(z_errmsg)[10] = { }; const char PREFIX3(vstring)[] = - " zlib-ng 2.2.1"; + " zlib-ng 2.2.4"; #ifdef ZLIB_COMPAT const char * Z_EXPORT zlibVersion(void) { diff --git a/src/runtime/src/native/external/zlib-ng/zutil_p.h b/src/runtime/src/native/external/zlib-ng/zutil_p.h index 97799f0ce31..835e12f4de5 100644 --- a/src/runtime/src/native/external/zlib-ng/zutil_p.h +++ b/src/runtime/src/native/external/zlib-ng/zutil_p.h @@ -43,33 +43,4 @@ static inline void zng_free(void *ptr) { #endif } -/* Use memcpy instead of memcmp to avoid older compilers not converting memcmp calls to - unaligned comparisons when unaligned access is supported. */ -static inline int32_t zng_memcmp_2(const void *src0, const void *src1) { - uint16_t src0_cmp, src1_cmp; - - memcpy(&src0_cmp, src0, sizeof(src0_cmp)); - memcpy(&src1_cmp, src1, sizeof(src1_cmp)); - - return src0_cmp != src1_cmp; -} - -static inline int32_t zng_memcmp_4(const void *src0, const void *src1) { - uint32_t src0_cmp, src1_cmp; - - memcpy(&src0_cmp, src0, sizeof(src0_cmp)); - memcpy(&src1_cmp, src1, sizeof(src1_cmp)); - - return src0_cmp != src1_cmp; -} - -static inline int32_t zng_memcmp_8(const void *src0, const void *src1) { - uint64_t src0_cmp, src1_cmp; - - memcpy(&src0_cmp, src0, sizeof(src0_cmp)); - memcpy(&src1_cmp, src1, sizeof(src1_cmp)); - - return src0_cmp != src1_cmp; -} - #endif diff --git a/src/runtime/src/tests/Directory.Build.targets b/src/runtime/src/tests/Directory.Build.targets index 23c3d94fe96..7546b5a10d6 100644 --- a/src/runtime/src/tests/Directory.Build.targets +++ b/src/runtime/src/tests/Directory.Build.targets @@ -574,8 +574,8 @@ - - + + diff --git a/src/runtime/src/tests/JIT/Regression/JitBlue/Runtime_115109/Runtime_115109.cs b/src/runtime/src/tests/JIT/Regression/JitBlue/Runtime_115109/Runtime_115109.cs new file mode 100644 index 00000000000..963c4543919 --- /dev/null +++ b/src/runtime/src/tests/JIT/Regression/JitBlue/Runtime_115109/Runtime_115109.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// Generated by Fuzzlyn v2.8 on 2025-04-27 22:46:08 +// Run on X86 Windows +// Seed: 2255917678885586044-vectort,vector128,vector256,x86aes,x86avx,x86avx2,x86avx512bw,x86avx512bwvl,x86avx512cd,x86avx512cdvl,x86avx512dq,x86avx512dqvl,x86avx512f,x86avx512fvl,x86bmi1,x86bmi2,x86fma,x86lzcnt,x86pclmulqdq,x86popcnt,x86sse,x86sse2,x86sse3,x86sse41,x86sse42,x86ssse3,x86x86base +// Reduced from 118.3 KiB to 1.0 KiB in 00:17:25 +// Debug: Outputs <0, 0, 0, 0, 0, 0, 0, 0> +// Release: Outputs <1, 0, 0, 0, 0, 0, 0, 0> +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using Xunit; + +public class Runtime_115109 +{ + static int s_5 = 101; + static Vector[] s_10; + static byte s_21; + + [Fact] + public static int TestEntryPoint() + { + s_10 = [Vector128.CreateScalar(0).AsVector()]; + bool vr5 = 0 <= M10(); + return s_10[0][0]; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static byte M10() + { + byte lvar2 = 3; + do + { + s_5 = 100; + int lvar4 = 0; + do + { + s_10[0] = Vector128.CreateScalar(s_5).AsVector(); + } + while (++lvar4 < 2); + } + while (--lvar2 > 1); + return s_21; + } +} diff --git a/src/runtime/src/tests/JIT/Regression/JitBlue/Runtime_115109/Runtime_115109.csproj b/src/runtime/src/tests/JIT/Regression/JitBlue/Runtime_115109/Runtime_115109.csproj new file mode 100644 index 00000000000..de6d5e08882 --- /dev/null +++ b/src/runtime/src/tests/JIT/Regression/JitBlue/Runtime_115109/Runtime_115109.csproj @@ -0,0 +1,8 @@ + + + True + + + + + diff --git a/src/runtime/src/tests/JIT/opt/SVE/PredicateInstructions.cs b/src/runtime/src/tests/JIT/opt/SVE/PredicateInstructions.cs index 6d460adbf4b..41b09c1fad3 100644 --- a/src/runtime/src/tests/JIT/opt/SVE/PredicateInstructions.cs +++ b/src/runtime/src/tests/JIT/opt/SVE/PredicateInstructions.cs @@ -15,18 +15,21 @@ public class PredicateInstructions [Fact] public static void TestPredicateInstructions() { - ZipLow(); - ZipHigh(); - UnzipOdd(); - UnzipEven(); - TransposeOdd(); - TransposeEven(); - ReverseElement(); - And(); - BitwiseClear(); - Xor(); - Or(); - ConditionalSelect(); + if (Sve.IsSupported) + { + ZipLow(); + ZipHigh(); + UnzipOdd(); + UnzipEven(); + TransposeOdd(); + TransposeEven(); + ReverseElement(); + And(); + BitwiseClear(); + Xor(); + Or(); + ConditionalSelect(); + } } [MethodImpl(MethodImplOptions.NoInlining)] diff --git a/src/source-manifest.json b/src/source-manifest.json index 5ee1e9ce83d..1023a9762ed 100644 --- a/src/source-manifest.json +++ b/src/source-manifest.json @@ -107,10 +107,10 @@ }, { "packageVersion": "10.0.0-preview.5.25262.10", - "barId": 269735, + "barId": 269898, "path": "runtime", "remoteUri": "https://github.com/dotnet/runtime", - "commitSha": "ab3d27fd2425f4403fcfbd04d93122d1f8bcb508" + "commitSha": "cb502bd97a0cac3f23c5c53ebda31238507213ee" }, { "packageVersion": "10.0.0-preview.25221.1", From 8fd93f7d47078163df7d03fafdfb4fd7427b9ae3 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 00:49:47 +0000 Subject: [PATCH 11/23] [main] Source code updates from dotnet/source-build-reference-packages (#856) Co-authored-by: dotnet-maestro[bot] Co-authored-by: Viktor Hofer --- .../eng/Version.Details.xml | 10 +++++----- src/source-build-reference-packages/eng/Versions.props | 2 +- .../eng/common/build.sh | 2 +- .../eng/common/tools.ps1 | 1 - .../eng/common/tools.sh | 2 -- src/source-build-reference-packages/global.json | 2 +- src/source-manifest.json | 4 ++-- 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/source-build-reference-packages/eng/Version.Details.xml b/src/source-build-reference-packages/eng/Version.Details.xml index 460d65ce27c..861e6281be4 100644 --- a/src/source-build-reference-packages/eng/Version.Details.xml +++ b/src/source-build-reference-packages/eng/Version.Details.xml @@ -1,14 +1,14 @@ - + - + https://github.com/dotnet/dotnet - f5705c8f4c5079bba77bae8698ba1583bde0388c + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - f5705c8f4c5079bba77bae8698ba1583bde0388c + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d https://github.com/dotnet/runtime diff --git a/src/source-build-reference-packages/eng/Versions.props b/src/source-build-reference-packages/eng/Versions.props index 526cb0fcb5a..643c1783b87 100644 --- a/src/source-build-reference-packages/eng/Versions.props +++ b/src/source-build-reference-packages/eng/Versions.props @@ -17,7 +17,7 @@ 6.0.0-preview.6.21352.12 6.0.0-preview.6.21352.12 - 10.0.100-preview.6.25276.103 + 10.0.100-preview.6.25277.102 1.4.13 diff --git a/src/source-build-reference-packages/eng/common/build.sh b/src/source-build-reference-packages/eng/common/build.sh index b105db583c4..9767bb411a4 100755 --- a/src/source-build-reference-packages/eng/common/build.sh +++ b/src/source-build-reference-packages/eng/common/build.sh @@ -257,7 +257,7 @@ function Build { /p:Sign=$sign \ /p:Publish=$publish \ /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \ - "${properties[@]}" + ${properties[@]+"${properties[@]}"} ExitWithExitCode 0 } diff --git a/src/source-build-reference-packages/eng/common/tools.ps1 b/src/source-build-reference-packages/eng/common/tools.ps1 index 046ec9d08a0..c9e39595b58 100644 --- a/src/source-build-reference-packages/eng/common/tools.ps1 +++ b/src/source-build-reference-packages/eng/common/tools.ps1 @@ -644,7 +644,6 @@ function GetNuGetPackageCachePath() { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' - $env:RESTORENOHTTPCACHE = $true } } diff --git a/src/source-build-reference-packages/eng/common/tools.sh b/src/source-build-reference-packages/eng/common/tools.sh index 8bc68e8460f..28944dfcb3f 100755 --- a/src/source-build-reference-packages/eng/common/tools.sh +++ b/src/source-build-reference-packages/eng/common/tools.sh @@ -345,14 +345,12 @@ function InitializeBuildTool { _InitializeBuildToolCommand="msbuild" } -# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages/" else export NUGET_PACKAGES="$repo_root/.packages/" - export RESTORENOHTTPCACHE=true fi fi diff --git a/src/source-build-reference-packages/global.json b/src/source-build-reference-packages/global.json index fbc9f4083a0..1a44a561f47 100644 --- a/src/source-build-reference-packages/global.json +++ b/src/source-build-reference-packages/global.json @@ -3,7 +3,7 @@ "dotnet": "10.0.100-preview.6.25272.112" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25276.103", + "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25277.102", "Microsoft.Build.NoTargets": "3.7.0" } } diff --git a/src/source-manifest.json b/src/source-manifest.json index 1023a9762ed..f9bceff2f80 100644 --- a/src/source-manifest.json +++ b/src/source-manifest.json @@ -135,10 +135,10 @@ }, { "packageVersion": "", - "barId": 269647, + "barId": 269844, "path": "source-build-reference-packages", "remoteUri": "https://github.com/dotnet/source-build-reference-packages", - "commitSha": "90b1fdf80b6fa1208ff386a2df6fe4528d0c3c6a" + "commitSha": "1cd3d5b3f12f47cfc3b3265b38efa84c28e5723b" }, { "packageVersion": "10.0.0-beta.25277.2", From a3d412f9f644f1cb9c906748246124146e568779 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 00:50:00 +0000 Subject: [PATCH 12/23] [main] Source code updates from dotnet/wpf (#863) Co-authored-by: dotnet-maestro[bot] Co-authored-by: Viktor Hofer --- src/source-manifest.json | 6 +- src/wpf/eng/Version.Details.xml | 102 ++++++++++++++++---------------- src/wpf/eng/Versions.props | 46 +++++++------- src/wpf/eng/common/build.sh | 2 +- src/wpf/eng/common/tools.ps1 | 1 - src/wpf/eng/common/tools.sh | 2 - src/wpf/global.json | 4 +- 7 files changed, 80 insertions(+), 83 deletions(-) diff --git a/src/source-manifest.json b/src/source-manifest.json index f9bceff2f80..14cbe8f5ad6 100644 --- a/src/source-manifest.json +++ b/src/source-manifest.json @@ -183,11 +183,11 @@ "commitSha": "9db17fd379a4b2f36f6ac274b4700fafd4713876" }, { - "packageVersion": "10.0.0-preview.6.25277.1", - "barId": 269663, + "packageVersion": "10.0.0-preview.6.25277.2", + "barId": 269754, "path": "wpf", "remoteUri": "https://github.com/dotnet/wpf", - "commitSha": "061f0464d39e2d7fc6a5cee2c1470f1a9cd025ee" + "commitSha": "cf1564f40b7ab8c0bac696f3405488fbd771adfb" }, { "packageVersion": "10.0.0-preview.25269.1", diff --git a/src/wpf/eng/Version.Details.xml b/src/wpf/eng/Version.Details.xml index 55ecdbe615e..2fe1356c714 100644 --- a/src/wpf/eng/Version.Details.xml +++ b/src/wpf/eng/Version.Details.xml @@ -1,112 +1,112 @@ - + - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d https://dev.azure.com/dnceng/internal/_git/dotnet-wpf-int 462eb030906d5cfcbcbbe22126855da1073cda15 - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d - + https://github.com/dotnet/dotnet - 9d86ce59f05e31b1bccb0cd5604f803416796fe4 + 57b0396ae0d21b9f0dfe0d208c57822fb88f9a8d diff --git a/src/wpf/eng/Versions.props b/src/wpf/eng/Versions.props index 5dbbfd699e3..7df00d527c3 100644 --- a/src/wpf/eng/Versions.props +++ b/src/wpf/eng/Versions.props @@ -19,35 +19,35 @@ dotnet/winforms is handling versions for the analyzers. --> $(MajorVersion).$(MinorVersion).0.0 - 10.0.0-beta.25274.104 - 10.0.0-beta.25274.104 - 10.0.0-beta.25274.104 + 10.0.0-beta.25277.102 + 10.0.0-beta.25277.102 + 10.0.0-beta.25277.102 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 4.6.0-preview4.19176.11 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 - 10.0.0-preview.6.25274.104 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 + 10.0.0-preview.6.25277.102 @@ -61,7 +61,7 @@ 9.0.0-beta.24053.1 - 10.0.0-beta.25274.104 + 10.0.0-beta.25277.102 diff --git a/src/wpf/eng/common/build.sh b/src/wpf/eng/common/build.sh index b105db583c4..9767bb411a4 100755 --- a/src/wpf/eng/common/build.sh +++ b/src/wpf/eng/common/build.sh @@ -257,7 +257,7 @@ function Build { /p:Sign=$sign \ /p:Publish=$publish \ /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \ - "${properties[@]}" + ${properties[@]+"${properties[@]}"} ExitWithExitCode 0 } diff --git a/src/wpf/eng/common/tools.ps1 b/src/wpf/eng/common/tools.ps1 index 046ec9d08a0..c9e39595b58 100644 --- a/src/wpf/eng/common/tools.ps1 +++ b/src/wpf/eng/common/tools.ps1 @@ -644,7 +644,6 @@ function GetNuGetPackageCachePath() { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' - $env:RESTORENOHTTPCACHE = $true } } diff --git a/src/wpf/eng/common/tools.sh b/src/wpf/eng/common/tools.sh index 8bc68e8460f..28944dfcb3f 100755 --- a/src/wpf/eng/common/tools.sh +++ b/src/wpf/eng/common/tools.sh @@ -345,14 +345,12 @@ function InitializeBuildTool { _InitializeBuildToolCommand="msbuild" } -# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages/" else export NUGET_PACKAGES="$repo_root/.packages/" - export RESTORENOHTTPCACHE=true fi fi diff --git a/src/wpf/global.json b/src/wpf/global.json index 79e133397f2..cc55aa7a424 100644 --- a/src/wpf/global.json +++ b/src/wpf/global.json @@ -14,8 +14,8 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25274.104", - "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.25274.104", + "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25277.102", + "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.25277.102", "Microsoft.Build.NoTargets": "3.7.56" }, "sdk": { From 251f5ef8926234f9d2b841ae7b1b72f4a68df5aa Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 00:50:17 +0000 Subject: [PATCH 13/23] [main] Source code updates from dotnet/sdk (#866) Co-authored-by: dotnet-maestro[bot] Co-authored-by: Viktor Hofer --- src/sdk/.config/dotnet-tools.json | 2 +- src/sdk/build/RunTestTemplateTests.ps1 | 46 +++++++++++++++++++ src/sdk/eng/Version.Details.xml | 16 +++---- src/sdk/eng/Versions.props | 6 +-- .../pipelines/templates/jobs/sdk-build.yml | 4 ++ src/sdk/sdk.sln | 7 ++- .../Microsoft.DotNet.ApiDiff.Tool/Program.cs | 42 +++++++++-------- .../FileOutputDiffGenerator.cs | 8 +++- .../IDiffGenerator.cs | 2 +- .../MemoryOutputDiffGenerator.cs | 6 ++- .../Diff.Base.Tests.cs | 2 +- .../Diff.Disk.Tests.cs | 12 ++--- src/source-manifest.json | 4 +- 13 files changed, 108 insertions(+), 49 deletions(-) create mode 100644 src/sdk/build/RunTestTemplateTests.ps1 diff --git a/src/sdk/.config/dotnet-tools.json b/src/sdk/.config/dotnet-tools.json index 8407115e314..4a4cd03b8bb 100644 --- a/src/sdk/.config/dotnet-tools.json +++ b/src/sdk/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dotnet.darc": { - "version": "1.1.0-beta.25272.3", + "version": "1.1.0-beta.25277.2", "commands": [ "darc" ] diff --git a/src/sdk/build/RunTestTemplateTests.ps1 b/src/sdk/build/RunTestTemplateTests.ps1 new file mode 100644 index 00000000000..5fe903b77f3 --- /dev/null +++ b/src/sdk/build/RunTestTemplateTests.ps1 @@ -0,0 +1,46 @@ +<# +.SYNOPSIS + Runs Microsoft.TestTemplates.Acceptance.Tests.dll in the dogfood environment. +.DESCRIPTION + This script enters the dogfood environment and runs the RunTestTemplateTests tests. +#> +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $configuration = "Release" +) + +function Run-TestTemplateTests { + $ErrorActionPreference = 'Stop' + $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..') + $classNameFilter = "--filter" + $filterValue = "FullyQualifiedName~Microsoft.DotNet.Cli.New.IntegrationTests.DotnetNewTestTemplatesTests" + $TestDll = Join-Path $RepoRoot "artifacts\bin\dotnet-new.IntegrationTests\$configuration\dotnet-new.IntegrationTests.dll" + + # Check if the test DLL exists + if (-not (Test-Path $TestDll)) { + Write-Error "Test DLL not found at: $TestDll" + return 1 + } + + Write-Host "Running tests for test templates in the dogfood environment..." -ForegroundColor Cyan + + # Call dogfood.ps1 directly instead of through dogfood.cmd to avoid the -NoExit parameter + $dogfoodPs1 = Join-Path $RepoRoot "eng\dogfood.ps1" + + Write-Host "Executing: dotnet test $TestDll via dogfood environment" -ForegroundColor Gray + # Pass the command directly to the dogfood.ps1 script + & $dogfoodPs1 -configuration $configuration -command @("dotnet", "test", $TestDll, $classNameFilter, $filterValue) + + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-Error "Tests failed with exit code: $exitCode" + } else { + Write-Host "Tests completed successfully!" -ForegroundColor Green + } + + return $exitCode +} + +# Execute the function using Invoke-Command +$exitCode = Invoke-Command -ScriptBlock ${function:Run-TestTemplateTests} +exit $exitCode diff --git a/src/sdk/eng/Version.Details.xml b/src/sdk/eng/Version.Details.xml index c2eae2626f8..124d6b7bcb5 100644 --- a/src/sdk/eng/Version.Details.xml +++ b/src/sdk/eng/Version.Details.xml @@ -538,21 +538,21 @@ https://github.com/dotnet/dotnet ad8565092bbfdd5c8b4a94a718d10b2d394f7aee - + https://github.com/dotnet/arcade-services - b6c1947b3493a1ad3a1a87fa689128e1d73309ba + 55b90ec8fea44527a4ab211cfe3ef50ccd608f74 - + https://github.com/dotnet/arcade-services - b6c1947b3493a1ad3a1a87fa689128e1d73309ba + 55b90ec8fea44527a4ab211cfe3ef50ccd608f74 - + https://github.com/microsoft/testfx - 65f8636524b4ffd6f7a6cd842714645f46905136 + 77c0915dfca9d2f0e9acdbced9be391d4f7f1dd9 - + https://github.com/microsoft/testfx - 65f8636524b4ffd6f7a6cd842714645f46905136 + 77c0915dfca9d2f0e9acdbced9be391d4f7f1dd9 https://github.com/dotnet/dotnet diff --git a/src/sdk/eng/Versions.props b/src/sdk/eng/Versions.props index 3c05a40e67f..34989c49bd3 100644 --- a/src/sdk/eng/Versions.props +++ b/src/sdk/eng/Versions.props @@ -30,7 +30,7 @@ true true - 1.8.0-preview.25276.8 + 1.8.0-preview.25277.4 30 @@ -89,7 +89,7 @@ - 1.1.0-beta.25272.3 + 1.1.0-beta.25277.2 @@ -322,7 +322,7 @@ 8.0.2 8.0.0 4.18.4 - 3.10.0-preview.25276.8 + 3.10.0-preview.25277.4 1.3.2 8.0.0-beta.23607.1 diff --git a/src/sdk/eng/pipelines/templates/jobs/sdk-build.yml b/src/sdk/eng/pipelines/templates/jobs/sdk-build.yml index fec0a54771d..14380f4682d 100644 --- a/src/sdk/eng/pipelines/templates/jobs/sdk-build.yml +++ b/src/sdk/eng/pipelines/templates/jobs/sdk-build.yml @@ -104,6 +104,10 @@ jobs: env: BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} + + - powershell: build/RunTestTemplateTests.ps1 + displayName: 🟣 Run Test Templates Tests + - ${{ else }}: - script: | source $(Build.SourcesDirectory)/eng/common/native/init-os-and-arch.sh diff --git a/src/sdk/sdk.sln b/src/sdk/sdk.sln index 43de36ddd32..9d90c45efbf 100644 --- a/src/sdk/sdk.sln +++ b/src/sdk/sdk.sln @@ -4,7 +4,6 @@ VisualStudioVersion = 17.1.31903.286 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{50A89C27-BA35-44B2-AC57-E54551791C64}" ProjectSection(SolutionItems) = preProject - eng\cgmanifest.json = eng\cgmanifest.json eng\dogfood.cmd = eng\dogfood.cmd eng\dogfood.sh = eng\dogfood.sh eng\Versions.props = eng\Versions.props @@ -1208,6 +1207,9 @@ Global {D7495CE7-64E5-4715-9304-799A41EC1D71} = {580D1AE7-AA8F-4912-8B76-105594E00B3B} {8D6A9984-118D-4415-A8FA-AB1F26CF5C44} = {71C279BD-E850-4A8D-9775-11CA26B8E5BA} {0CBA5FB8-71A3-457A-89F3-E52B9602164A} = {71C279BD-E850-4A8D-9775-11CA26B8E5BA} + {3C688B3B-7919-6C9E-93A3-A3F6B3E684A9} = {22AB674F-ED91-4FBC-BFEE-8A1E82F9F05E} + {48A69DF9-9233-2548-C614-202D5780014D} = {3C688B3B-7919-6C9E-93A3-A3F6B3E684A9} + {0D36F844-0A1C-469F-93A1-C0E5AD141B07} = {48A69DF9-9233-2548-C614-202D5780014D} {21C21975-84C1-4A24-8E21-F7EC790A4584} = {580D1AE7-AA8F-4912-8B76-105594E00B3B} {19014C60-F87C-4CC7-AC0F-C41B6126EBCE} = {71A9F549-0EB6-41F9-BC16-4A6C5007FC91} {94C8526E-DCC2-442F-9868-3DD0BA2688BE} = {71A9F549-0EB6-41F9-BC16-4A6C5007FC91} @@ -1222,9 +1224,6 @@ Global {FA3C7F91-42A2-45AD-897C-F646B081016C} = {71A9F549-0EB6-41F9-BC16-4A6C5007FC91} {3DF5A9B8-6F90-4CFB-4518-0E97982B6748} = {71A9F549-0EB6-41F9-BC16-4A6C5007FC91} {0762B436-F4B0-4008-9097-BB5FF6BD84AF} = {71A9F549-0EB6-41F9-BC16-4A6C5007FC91} - {3C688B3B-7919-6C9E-93A3-A3F6B3E684A9} = {22AB674F-ED91-4FBC-BFEE-8A1E82F9F05E} - {48A69DF9-9233-2548-C614-202D5780014D} = {3C688B3B-7919-6C9E-93A3-A3F6B3E684A9} - {0D36F844-0A1C-469F-93A1-C0E5AD141B07} = {48A69DF9-9233-2548-C614-202D5780014D} {8D2921DD-ED7E-68AB-522C-54E23AAAFFCC} = {3C688B3B-7919-6C9E-93A3-A3F6B3E684A9} {485FC8E4-6776-43E4-AD87-C583BC6136FE} = {8D2921DD-ED7E-68AB-522C-54E23AAAFFCC} {27BBE29B-CE6F-401F-B3CF-B07DC556FAD1} = {22AB674F-ED91-4FBC-BFEE-8A1E82F9F05E} diff --git a/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff.Tool/Program.cs b/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff.Tool/Program.cs index a8955d1f2f9..0a8f5cbf47e 100644 --- a/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff.Tool/Program.cs +++ b/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff.Tool/Program.cs @@ -18,66 +18,66 @@ public static class Program public static async Task Main(string[] args) { - RootCommand rootCommand = new("genapidiff"); + RootCommand rootCommand = new("ApiDiff - Tool for generating a markdown diff of two different versions of the same assembly."); - Option optionBeforeAssembliesFolderPath = new(name: "", aliases: ["--before", "-b"]) + Option optionBeforeAssembliesFolderPath = new("--before", "-b") { Description = "The path to the folder containing the old (before) assemblies to be included in the diff.", Arity = ArgumentArity.ExactlyOne, Required = true }; - Option optionBeforeRefAssembliesFolderPath = new(name: "", aliases: ["--refbefore", "-rb"]) + Option optionBeforeRefAssembliesFolderPath = new("--refbefore", "-rb") { Description = "The path to the folder containing the references required by old (before) assemblies, not to be included in the diff.", Arity = ArgumentArity.ExactlyOne, Required = false }; - Option optionAfterAssembliesFolderPath = new(name: "", aliases: ["--after", "-a"]) + Option optionAfterAssembliesFolderPath = new("--after", "-a") { Description = "The path to the folder containing the new (after) assemblies to be included in the diff.", Arity = ArgumentArity.ExactlyOne, Required = true }; - Option optionAfterRefAssembliesFolderPath = new(name: "", aliases: ["--refafter", "-ra"]) + Option optionAfterRefAssembliesFolderPath = new("--refafter", "-ra") { Description = "The path to the folder containing references required by the new (after) reference assemblies, not to be included in the diff.", Arity = ArgumentArity.ExactlyOne, Required = false }; - Option optionOutputFolderPath = new(name: "", aliases: ["--output", "-o"]) + Option optionOutputFolderPath = new("--output", "-o") { Description = "The path to the output folder.", Arity = ArgumentArity.ExactlyOne, Required = true }; - Option optionBeforeFriendlyName = new(name: "", aliases: ["--beforeFriendlyName", "-bfn"]) + Option optionBeforeFriendlyName = new("--beforeFriendlyName", "-bfn") { Description = "The friendly name to describe the 'before' assembly.", Arity = ArgumentArity.ExactlyOne, Required = true }; - Option optionAfterFriendlyName = new(name: "", aliases: ["--afterFriendlyName", "-afn"]) + Option optionAfterFriendlyName = new("--afterFriendlyName", "-afn") { Description = "The friendly name to describe the 'after' assembly.", Arity = ArgumentArity.ExactlyOne, Required = true }; - Option optionTableOfContentsTitle = new(name: "", aliases: ["--tableOfContentsTitle", "-tc"]) + Option optionTableOfContentsTitle = new("--tableOfContentsTitle", "-tc") { Description = $"The optional title of the markdown table of contents file that is placed in the output folder.", - Arity = ArgumentArity.ZeroOrMore, + Arity = ArgumentArity.ExactlyOne, Required = false, DefaultValueFactory = _ => "api_diff" }; - Option optionFilesWithAssembliesToExclude = new(name: "", aliases: ["--assembliesToExclude", "-eas"]) + Option optionFilesWithAssembliesToExclude = new("--assembliesToExclude", "-eas") { Description = "An optional array of filepaths, each containing a list of assemblies that should be excluded from the diff. Each file should contain one assembly name per line, with no extensions.", Arity = ArgumentArity.ZeroOrMore, @@ -85,7 +85,7 @@ public static async Task Main(string[] args) DefaultValueFactory = _ => null }; - Option optionFilesWithAttributesToExclude = new(name: "", aliases: ["--attributesToExclude", "-eattrs"]) + Option optionFilesWithAttributesToExclude = new("--attributesToExclude", "-eattrs") { Description = $"An optional array of filepaths, each containing a list of attributes to exclude from the diff. Each file should contain one API full name per line. You can either modify the default file '{AttributesToExcludeDefaultFileName}' to add your own attributes, or include additional files using this command line option.", Arity = ArgumentArity.ZeroOrMore, @@ -93,7 +93,7 @@ public static async Task Main(string[] args) DefaultValueFactory = _ => [new FileInfo(AttributesToExcludeDefaultFileName)] }; - Option optionFilesWithApisToExclude = new(name: "", aliases: ["--apisToExclude", "-eapis"]) + Option optionFilesWithApisToExclude = new("--apisToExclude", "-eapis") { Description = "An optional array of filepaths, each containing a list of APIs to exclude from the diff. Each file should contain one API full name per line.", Arity = ArgumentArity.ZeroOrMore, @@ -101,13 +101,13 @@ public static async Task Main(string[] args) DefaultValueFactory = _ => null }; - Option optionAddPartialModifier = new(name: "", aliases: ["--addPartialModifier", "-apm"]) + Option optionAddPartialModifier = new("--addPartialModifier", "-apm") { Description = "Add the 'partial' modifier to types.", DefaultValueFactory = _ => false }; - Option optionAttachDebugger = new(name: "", aliases: ["--attachDebugger", "-d"]) + Option optionAttachDebugger = new("--attachDebugger", "-d") { Description = "Stops the tool at startup, prints the process ID and waits for a debugger to attach.", DefaultValueFactory = _ => false @@ -128,9 +128,9 @@ public static async Task Main(string[] args) rootCommand.Options.Add(optionAddPartialModifier); rootCommand.Options.Add(optionAttachDebugger); - rootCommand.SetAction(async (ParseResult result) => + rootCommand.SetAction(async (ParseResult result, CancellationToken cancellationToken) => { - DiffConfiguration c = new( + DiffConfiguration diffConfig = new( BeforeAssembliesFolderPath: result.GetValue(optionBeforeAssembliesFolderPath) ?? throw new NullReferenceException("Null before assemblies directory"), BeforeAssemblyReferencesFolderPath: result.GetValue(optionBeforeRefAssembliesFolderPath), AfterAssembliesFolderPath: result.GetValue(optionAfterAssembliesFolderPath) ?? throw new NullReferenceException("Null after assemblies directory"), @@ -145,13 +145,15 @@ public static async Task Main(string[] args) AddPartialModifier: result.GetValue(optionAddPartialModifier), AttachDebugger: result.GetValue(optionAttachDebugger) ); - await HandleCommandAsync(c).ConfigureAwait(false); + await HandleCommandAsync(diffConfig, cancellationToken).ConfigureAwait(false); }); await rootCommand.Parse(args).InvokeAsync(); } - private static Task HandleCommandAsync(DiffConfiguration diffConfig) + private static Task HandleCommandAsync(DiffConfiguration diffConfig, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + var log = new ConsoleLog(MessageImportance.Normal); string assembliesToExclude = string.Join(", ", diffConfig.FilesWithAssembliesToExclude?.Select(a => a.FullName) ?? []); @@ -197,7 +199,7 @@ private static Task HandleCommandAsync(DiffConfiguration diffConfig) diagnosticOptions: null // TODO: If needed, add CLI option to pass specific diagnostic options ); - return diffGenerator.RunAsync(); + return diffGenerator.RunAsync(cancellationToken); } private static void WaitForDebugger() diff --git a/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/FileOutputDiffGenerator.cs b/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/FileOutputDiffGenerator.cs index b86a24d77cd..ae5ef036776 100644 --- a/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/FileOutputDiffGenerator.cs +++ b/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/FileOutputDiffGenerator.cs @@ -87,11 +87,13 @@ internal FileOutputDiffGenerator(ILog log, public IReadOnlyDictionary Results => _results.AsReadOnly(); /// - public async Task RunAsync() + public async Task RunAsync(CancellationToken cancellationToken) { Debug.Assert(_beforeAssembliesFolderPaths.Length == 1); Debug.Assert(_afterAssembliesFolderPaths.Length == 1); + cancellationToken.ThrowIfCancellationRequested(); + (IAssemblySymbolLoader beforeLoader, Dictionary beforeAssemblySymbols) = AssemblySymbolLoader.CreateFromFiles( _log, @@ -118,7 +120,7 @@ public async Task RunAsync() _addPartialModifier, _diagnosticOptions); - await generator.RunAsync().ConfigureAwait(false); + await generator.RunAsync(cancellationToken).ConfigureAwait(false); // If true, output is disk. Otherwise, it's the Results dictionary. if (_writeToDisk) @@ -135,6 +137,8 @@ public async Task RunAsync() foreach ((string assemblyName, string text) in generator.Results.OrderBy(r => r.Key)) { + cancellationToken.ThrowIfCancellationRequested(); + string fileName = $"{_tableOfContentsTitle}_{assemblyName}.md"; tableOfContents.AppendLine($"* [{assemblyName}]({fileName})"); diff --git a/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/IDiffGenerator.cs b/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/IDiffGenerator.cs index 32c7f2ceb47..7a27f4c5bb9 100644 --- a/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/IDiffGenerator.cs +++ b/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/IDiffGenerator.cs @@ -13,5 +13,5 @@ public interface IDiffGenerator /// /// Asynchronously runs the diff generator and may populate the dictionary depending on the use case. /// - Task RunAsync(); + Task RunAsync(CancellationToken cancellationToken); } diff --git a/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/MemoryOutputDiffGenerator.cs b/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/MemoryOutputDiffGenerator.cs index a5cb01b16ef..a481547425f 100644 --- a/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/MemoryOutputDiffGenerator.cs +++ b/src/sdk/src/Compatibility/ApiDiff/Microsoft.DotNet.ApiDiff/MemoryOutputDiffGenerator.cs @@ -85,12 +85,14 @@ internal MemoryOutputDiffGenerator( public IReadOnlyDictionary Results => _results.AsReadOnly(); /// - public async Task RunAsync() + public async Task RunAsync(CancellationToken cancellationToken) { Stopwatch swRun = Stopwatch.StartNew(); foreach ((string beforeAssemblyName, IAssemblySymbol beforeAssemblySymbol) in _beforeAssemblySymbols) { + cancellationToken.ThrowIfCancellationRequested(); + // Needs to block so the _afterAssemblySymbols dictionary gets updated. await ProcessBeforeAndAfterAssemblyAsync(beforeAssemblyName, beforeAssemblySymbol).ConfigureAwait(false); } @@ -98,6 +100,8 @@ public async Task RunAsync() // Needs to happen after processing the before and after assemblies and filtering out the existing ones. foreach ((string afterAssemblyName, IAssemblySymbol afterAssemblySymbol) in _afterAssemblySymbols) { + cancellationToken.ThrowIfCancellationRequested(); + await ProcessNewAssemblyAsync(afterAssemblyName, afterAssemblySymbol).ConfigureAwait(false); } diff --git a/src/sdk/test/Microsoft.DotNet.ApiDiff.Tests/Diff.Base.Tests.cs b/src/sdk/test/Microsoft.DotNet.ApiDiff.Tests/Diff.Base.Tests.cs index d4d9f893b80..55d47042790 100644 --- a/src/sdk/test/Microsoft.DotNet.ApiDiff.Tests/Diff.Base.Tests.cs +++ b/src/sdk/test/Microsoft.DotNet.ApiDiff.Tests/Diff.Base.Tests.cs @@ -61,7 +61,7 @@ protected async Task RunTestAsync( addPartialModifier, DiffGeneratorFactory.DefaultDiagnosticOptions); - await generator.RunAsync(); + await generator.RunAsync(CancellationToken.None); foreach ((string expectedAssemblyName, string expectedCode) in expected) { diff --git a/src/sdk/test/Microsoft.DotNet.ApiDiff.Tests/Diff.Disk.Tests.cs b/src/sdk/test/Microsoft.DotNet.ApiDiff.Tests/Diff.Disk.Tests.cs index cb699c33626..24e96ea62a2 100644 --- a/src/sdk/test/Microsoft.DotNet.ApiDiff.Tests/Diff.Disk.Tests.cs +++ b/src/sdk/test/Microsoft.DotNet.ApiDiff.Tests/Diff.Disk.Tests.cs @@ -96,7 +96,7 @@ public async Task DiskRead_DiskWrite() outputFolderPath.DirPath, writeToDisk: true); - await generator.RunAsync(); + await generator.RunAsync(CancellationToken.None); VerifyDiskWrite(outputFolderPath.DirPath, DefaultTableOfContentsTitle, ExpectedTableOfContents, DefaultExpectedAssemblyMarkdowns); } @@ -255,7 +255,7 @@ Lines preceded by a '+' are additions and a '-' indicates removal. outputFolderPath.DirPath, writeToDisk: true); - await generator.RunAsync(); + await generator.RunAsync(CancellationToken.None); VerifyDiskWrite(outputFolderPath.DirPath, DefaultTableOfContentsTitle, expectedTableOfContents, expectedAssemblyMarkdowns); } @@ -286,7 +286,7 @@ public async Task DiskRead_DiskWrite_ExcludeAssembly() writeToDisk: true, filesWithAssembliesToExclude: await GetFileWithListsAsync(root, [DefaultAssemblyName])); - await generator.RunAsync(); + await generator.RunAsync(CancellationToken.None); VerifyDiskWrite(outputFolderPath.FullName, DefaultTableOfContentsTitle, ExpectedEmptyTableOfContents, []); } @@ -376,7 +376,7 @@ public class MySubClass outputFolderPath.DirPath, writeToDisk: true); - await generator.RunAsync(); + await generator.RunAsync(CancellationToken.None); VerifyDiskWrite(outputFolderPath.DirPath, DefaultTableOfContentsTitle, ExpectedTableOfContents, expectedAssemblyMarkdowns); } @@ -401,7 +401,7 @@ public async Task DiskRead_MemoryWrite() outputFolderPath.DirPath, writeToDisk: false); - await generator.RunAsync(); + await generator.RunAsync(CancellationToken.None); string tableOfContentsMarkdownFilePath = Path.Join(outputFolderPath.DirPath, $"{DefaultTableOfContentsTitle}.md"); Assert.Contains(tableOfContentsMarkdownFilePath, generator.Results.Keys); @@ -440,7 +440,7 @@ public async Task DiskRead_MemoryWrite_ExcludeAssembly() writeToDisk: false, filesWithAssembliesToExclude: await GetFileWithListsAsync(root, [DefaultAssemblyName])); - await generator.RunAsync(); + await generator.RunAsync(CancellationToken.None); string tableOfContentsMarkdownFilePath = Path.Join(outputFolderPath.FullName, $"{DefaultTableOfContentsTitle}.md"); Assert.Contains(tableOfContentsMarkdownFilePath, generator.Results.Keys); diff --git a/src/source-manifest.json b/src/source-manifest.json index 14cbe8f5ad6..ed44cd7cc04 100644 --- a/src/source-manifest.json +++ b/src/source-manifest.json @@ -121,10 +121,10 @@ }, { "packageVersion": "10.0.100-preview.6.25272.9", - "barId": 269694, + "barId": 269876, "path": "sdk", "remoteUri": "https://github.com/dotnet/sdk", - "commitSha": "39c03282e67a91f4e2c71cf56e69271afb860d37" + "commitSha": "1dd8c27e8a5bcdebe57517c5ed042645afe25692" }, { "packageVersion": "10.0.622801", From cecde05d9c87789c79504c9852a6c0ab4058a5b7 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 00:50:41 +0000 Subject: [PATCH 14/23] [main] Source code updates from dotnet/fsharp (#860) Co-authored-by: dotnet-maestro[bot] Co-authored-by: Viktor Hofer --- src/fsharp/eng/DotNetBuild.props | 2 +- src/fsharp/eng/Version.Details.xml | 2 +- src/source-manifest.json | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fsharp/eng/DotNetBuild.props b/src/fsharp/eng/DotNetBuild.props index 4c243eff3e8..b78d9712948 100644 --- a/src/fsharp/eng/DotNetBuild.props +++ b/src/fsharp/eng/DotNetBuild.props @@ -28,7 +28,7 @@ -bl enables the binlogs for the tools and Proto builds, which make debugging failures here easier --> diff --git a/src/fsharp/eng/Version.Details.xml b/src/fsharp/eng/Version.Details.xml index 612b8e5a32f..f68624a4cd6 100644 --- a/src/fsharp/eng/Version.Details.xml +++ b/src/fsharp/eng/Version.Details.xml @@ -1,6 +1,6 @@ - + https://github.com/dotnet/source-build-reference-packages diff --git a/src/source-manifest.json b/src/source-manifest.json index ed44cd7cc04..9bde5e3749b 100644 --- a/src/source-manifest.json +++ b/src/source-manifest.json @@ -64,11 +64,11 @@ "commitSha": "80d3bcc1ac4785e6c5e348a9caf6125873ecb30d" }, { - "packageVersion": "10.0.100-beta.25270.2", - "barId": 269002, + "packageVersion": "10.0.100-beta.25278.2", + "barId": 269887, "path": "fsharp", "remoteUri": "https://github.com/dotnet/fsharp", - "commitSha": "13ad6469b0354735c5b259f7e8307648ab7a6c50" + "commitSha": "79ced2b7b6c4a850507fb6ffdc75ab6ffb5e0146" }, { "packageVersion": "17.15.0-preview-25277-08", From a60cdec70c51d614d575bb029febf415c0b82d3e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 00:50:46 +0000 Subject: [PATCH 15/23] [main] Source code updates from dotnet/razor (#861) Co-authored-by: dotnet-maestro[bot] Co-authored-by: Viktor Hofer --- .../legacyTest/Legacy/RazorParserTest.cs | 2 +- .../DefaultRazorTagHelperBinderPhaseTest.cs | 26 +- .../FindTokenIntegrationTest.cs | 2 +- .../test/Legacy/RazorParserTest.cs | 3 +- .../test/Syntax/FindTokenTests.cs | 132 +++--- .../DefaultDirectiveSyntaxTreePass.cs | 4 +- ...aultRazorTagHelperContextDiscoveryPhase.cs | 118 ++--- .../Language/Legacy/TagHelperBlockRewriter.cs | 2 +- .../Legacy/TagHelperParseTreeRewriter.cs | 7 +- .../Language/ReadOnlyMemoryOfCharComparer.cs | 42 ++ .../src/Language/Syntax/RazorSyntaxNode.cs | 28 +- .../src/Language/Syntax/SyntaxList.cs | 36 +- .../Language/Syntax/SyntaxNode.Serializer.cs | 36 ++ .../src/Language/Syntax/SyntaxNode.cs | 32 +- .../Language/Syntax/SyntaxNodeExtensions.cs | 48 +- .../src/Language/Syntax/SyntaxReplacer.cs | 282 +++++++++--- .../src/Language/Syntax/SyntaxRewriter.cs | 65 ++- .../src/Language/Syntax/SyntaxSerializer.cs | 425 ++++++++---------- .../src/Language/Syntax/SyntaxToken.cs | 36 +- .../src/Language/Syntax/SyntaxVisitor.cs | 47 +- .../src/Language/Syntax/SyntaxWalker.cs | 37 +- .../src/Language/TagHelperDescriptor.cs | 32 +- .../AbstractRazorCompletionFactsService.cs | 11 + .../MarkupTransitionCompletionItemProvider.cs | 2 +- .../Extensions/RazorSyntaxNodeExtensions.cs | 2 +- ...pFormattingPass.CSharpDocumentGenerator.cs | 2 +- .../SemanticTokens/SemanticTokensVisitor.cs | 23 +- .../Language/Legacy/ToolingParserTestBase.cs | 4 +- .../CompletionIntegrationTests.cs | 23 + .../Language/Legacy/ParserTestBase.cs | 4 +- .../Language/Legacy/SyntaxNodeSerializer.cs | 62 --- .../Language/Legacy/SyntaxNodeWalker.cs | 38 -- .../Language/Legacy/SyntaxNodeWriter.cs | 237 ---------- .../Language/TestSyntaxSerializer.cs | 72 +++ src/source-manifest.json | 6 +- 35 files changed, 1001 insertions(+), 927 deletions(-) create mode 100644 src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/ReadOnlyMemoryOfCharComparer.cs create mode 100644 src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNode.Serializer.cs delete mode 100644 src/razor/src/Shared/Microsoft.AspNetCore.Razor.Test.Common/Language/Legacy/SyntaxNodeSerializer.cs delete mode 100644 src/razor/src/Shared/Microsoft.AspNetCore.Razor.Test.Common/Language/Legacy/SyntaxNodeWalker.cs delete mode 100644 src/razor/src/Shared/Microsoft.AspNetCore.Razor.Test.Common/Language/Legacy/SyntaxNodeWriter.cs create mode 100644 src/razor/src/Shared/Microsoft.AspNetCore.Razor.Test.Common/Language/TestSyntaxSerializer.cs diff --git a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/RazorParserTest.cs b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/RazorParserTest.cs index a6c9bbd554c..27e08d0dbeb 100644 --- a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/RazorParserTest.cs +++ b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/legacyTest/Legacy/RazorParserTest.cs @@ -37,7 +37,7 @@ public void ParseMethodCallsParseDocumentOnMarkupParserAndReturnsResults() var syntaxTree = parser.Parse(TestRazorSourceDocument.Create("foo @bar baz")); // Assert - var actual = SyntaxNodeSerializer.Serialize(syntaxTree.Root, validateSpanEditHandlers: true); + var actual = TestSyntaxSerializer.Serialize(syntaxTree.Root, allowSpanEditHandlers: true); AssertEx.AssertEqualToleratingWhitespaceDifferences(expected, actual); } } diff --git a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/DefaultRazorTagHelperBinderPhaseTest.cs b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/DefaultRazorTagHelperBinderPhaseTest.cs index abea6f12392..b50d579fc1d 100644 --- a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/DefaultRazorTagHelperBinderPhaseTest.cs +++ b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/DefaultRazorTagHelperBinderPhaseTest.cs @@ -1245,26 +1245,6 @@ @using static SomeProject.SomeOtherFolder.Foo Assert.Same(componentDescriptor, result); } - [Theory] - [InlineData("", "", true)] - [InlineData("Foo", "Project", true)] - [InlineData("Project.Foo", "Project", true)] - [InlineData("Project.Foo", "global::Project", true)] - [InlineData("Project.Bar.Foo", "Project.Bar", true)] - [InlineData("Project.Foo", "Project.Bar", false)] - [InlineData("Project.Foo", "global::Project.Bar", false)] - [InlineData("Project.Bar.Foo", "Project", false)] - [InlineData("Bar.Foo", "Project", false)] - public void IsTypeInNamespace_WorksAsExpected(string typeName, string @namespace, bool expected) - { - // Arrange & Act - var descriptor = CreateComponentDescriptor(typeName, typeName, "Test.dll"); - var result = DefaultRazorTagHelperContextDiscoveryPhase.ComponentDirectiveVisitor.IsTypeInNamespace(descriptor, @namespace); - - // Assert - Assert.Equal(expected, result); - } - [Theory] [InlineData("", "", true)] [InlineData("Foo", "Project", true)] @@ -1273,11 +1253,13 @@ public void IsTypeInNamespace_WorksAsExpected(string typeName, string @namespace [InlineData("Project.Foo", "Project.Bar", true)] [InlineData("Project.Bar.Foo", "Project", false)] [InlineData("Bar.Foo", "Project", false)] - public void IsTypeInScope_WorksAsExpected(string typeName, string currentNamespace, bool expected) + public void IsTypeNamespaceInScope_WorksAsExpected(string typeName, string currentNamespace, bool expected) { // Arrange & Act var descriptor = CreateComponentDescriptor(typeName, typeName, "Test.dll"); - var result = DefaultRazorTagHelperContextDiscoveryPhase.ComponentDirectiveVisitor.IsTypeInScope(descriptor, currentNamespace); + var tagHelperTypeNamespace = descriptor.GetTypeNamespace(); + + var result = DefaultRazorTagHelperContextDiscoveryPhase.ComponentDirectiveVisitor.IsTypeNamespaceInScope(tagHelperTypeNamespace, currentNamespace); // Assert Assert.Equal(expected, result); diff --git a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/IntegrationTests/FindTokenIntegrationTest.cs b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/IntegrationTests/FindTokenIntegrationTest.cs index ce12732a233..545d59c9d7e 100644 --- a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/IntegrationTests/FindTokenIntegrationTest.cs +++ b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/IntegrationTests/FindTokenIntegrationTest.cs @@ -20,6 +20,6 @@ public void EmptyDirective() var root = codeDocument.GetSyntaxTree().Root; var token = root.FindToken(27); - AssertEx.Equal("Identifier;[];", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("Identifier;[];", TestSyntaxSerializer.Serialize(token).Trim()); } } diff --git a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/RazorParserTest.cs b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/RazorParserTest.cs index 85c24eaae1b..98316708891 100644 --- a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/RazorParserTest.cs +++ b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/RazorParserTest.cs @@ -3,6 +3,7 @@ #nullable disable +using Microsoft.AspNetCore.Razor.Language; using Xunit; namespace Microsoft.AspNetCore.Razor.Language.Legacy; @@ -48,7 +49,7 @@ public void ParseMethodCallsParseDocumentOnMarkupParserAndReturnsResults() var syntaxTree = parser.Parse(TestRazorSourceDocument.Create("foo @bar baz")); // Assert - var actual = SyntaxNodeSerializer.Serialize(syntaxTree.Root, validateSpanEditHandlers: false); + var actual = TestSyntaxSerializer.Serialize(syntaxTree.Root, allowSpanEditHandlers: false); Assert.Equal(expected, actual); } } diff --git a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Syntax/FindTokenTests.cs b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Syntax/FindTokenTests.cs index a0f5cd31dfd..22986df7432 100644 --- a/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Syntax/FindTokenTests.cs +++ b/src/razor/src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Syntax/FindTokenTests.cs @@ -39,7 +39,7 @@ public void ReturnsEofOnFileEnd() var token = tree.Root.FindToken(position); - AssertEx.Equal("""EndOfFile;[];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""EndOfFile;[];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact, WorkItem("https://github.com/dotnet/razor/issues/7505")] @@ -54,7 +54,7 @@ public void ReturnsEofOnFileEnd_WithTrailingTrivia() var token = tree.Root.FindToken(position); - AssertEx.Equal("""EndOfFile;[];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""EndOfFile;[];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact, WorkItem("https://github.com/dotnet/razor/issues/9040")] @@ -67,7 +67,7 @@ public void LeadingWhitespace_BeforeAnyNode() var token = tree.Root.FindToken(position); - AssertEx.Equal("""OpenAngle;[<];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""OpenAngle;[<];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -78,7 +78,7 @@ public void ReturnsOpenAngle() var token = tree.Root.FindToken(position); - AssertEx.Equal("""OpenAngle;[<];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""OpenAngle;[<];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Theory] @@ -91,7 +91,7 @@ public void ReturnsStartDivTag(string text) var token = tree.Root.FindToken(position); - AssertEx.Equal("""Text;[div];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Text;[div];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -102,7 +102,7 @@ public void ReturnsCloseAngle() var token = tree.Root.FindToken(position); - AssertEx.Equal("""CloseAngle;[>];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""CloseAngle;[>];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact, WorkItem("https://github.com/dotnet/razor/issues/7630")] @@ -113,7 +113,7 @@ public void ReturnsEof_AfterVoidTag() var token = tree.Root.FindToken(position); - AssertEx.Equal("""EndOfFile;[];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""EndOfFile;[];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact, WorkItem("https://github.com/dotnet/razor/issues/7630")] @@ -124,7 +124,7 @@ public void ReturnsCloseAngle_AfterVoidTagWithTrailingSpace() var token = tree.Root.FindToken(position); - AssertEx.Equal("""CloseAngle;[>];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""CloseAngle;[>];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -139,7 +139,7 @@ public void CSharpTransition_01() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Transition;[@];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Transition;[@];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -154,7 +154,7 @@ public void CSharpTransition_02() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Keyword;[if];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Keyword;[if];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -169,7 +169,7 @@ public void CSharpTransition_03_IgnoreWhitespace() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Keyword;[if];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Keyword;[if];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -184,7 +184,7 @@ public void CSharpTransition_03_IncludeWhitespace() var token = tree.Root.FindToken(position, includeWhitespace: true); - AssertEx.Equal("""Whitespace;[ ];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Whitespace;[ ];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -199,7 +199,7 @@ public void CSharpTransition_04() var token = tree.Root.FindToken(position); - AssertEx.Equal("""LeftParenthesis;[(];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""LeftParenthesis;[(];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -214,7 +214,7 @@ public void CSharpTransition_05() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Keyword;[true];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Keyword;[true];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -229,7 +229,7 @@ public void CSharpTransition_06() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RightParenthesis;[)];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RightParenthesis;[)];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -244,7 +244,7 @@ public void CSharpTransition_07_IgnoreWhitespace() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RightParenthesis;[)];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RightParenthesis;[)];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -259,7 +259,7 @@ public void CSharpTransition_07_IncludeWhitespace() var token = tree.Root.FindToken(position, includeWhitespace: true); - AssertEx.Equal("""NewLine;[LF];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""NewLine;[LF];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -274,7 +274,7 @@ public void CSharpTransition_08() var token = tree.Root.FindToken(position); - AssertEx.Equal("""LeftBrace;[{];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""LeftBrace;[{];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -290,7 +290,7 @@ public void CSharpTransition_09() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RightBrace;[}];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RightBrace;[}];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -305,7 +305,7 @@ public void CSharpTransition_10() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RightBrace;[}];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RightBrace;[}];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -318,7 +318,7 @@ public void CSharpTransition_11() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Transition;[@];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Transition;[@];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -331,7 +331,7 @@ public void CSharpTransition_12() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Identifier;[value];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Identifier;[value];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -344,7 +344,7 @@ public void CSharpTransition_12_IgnoreWhitespace() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Identifier;[value];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Identifier;[value];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -357,7 +357,7 @@ public void CSharpTransition_12_IncludeWhitespace() var token = tree.Root.FindToken(position, includeWhitespace: true); - AssertEx.Equal("""Whitespace;[ ];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Whitespace;[ ];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -370,7 +370,7 @@ public void CSharpTransition_13() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Transition;[@];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Transition;[@];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -383,7 +383,7 @@ public void CSharpTransition_14() var token = tree.Root.FindToken(position); - AssertEx.Equal("""LeftParenthesis;[(];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""LeftParenthesis;[(];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -396,7 +396,7 @@ public void CSharpTransition_15() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Identifier;[value];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Identifier;[value];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -409,7 +409,7 @@ public void CSharpTransition_16() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RightParenthesis;[)];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RightParenthesis;[)];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -422,7 +422,7 @@ public void CSharpTransition_17_IgnoreWhitespace() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RightParenthesis;[)];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RightParenthesis;[)];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -435,7 +435,7 @@ public void CSharpTransition_17_IncludeWhitespace() var token = tree.Root.FindToken(position, includeWhitespace: true); - AssertEx.Equal("""Whitespace;[ ];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Whitespace;[ ];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -451,7 +451,7 @@ public void HtmlTransition_01() var token = tree.Root.FindToken(position); - AssertEx.Equal("""OpenAngle;[<];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""OpenAngle;[<];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Theory, WorkItem("https://github.com/dotnet/razor/issues/7630")] @@ -470,7 +470,7 @@ public void HtmlTransition_02(string tagContent) var token = tree.Root.FindToken(position); - AssertEx.Equal("""CloseAngle;[>];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""CloseAngle;[>];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact, WorkItem("https://github.com/dotnet/razor/issues/7630")] @@ -486,7 +486,7 @@ public void HtmlTransition_03() var token = tree.Root.FindToken(position); - AssertEx.Equal("""OpenAngle;[<];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""OpenAngle;[<];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact, WorkItem("https://github.com/dotnet/razor/issues/7630")] @@ -502,7 +502,7 @@ public void HtmlTransition_04() var token = tree.Root.FindToken(position); - AssertEx.Equal("""CloseAngle;[>];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""CloseAngle;[>];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Theory, WorkItem("https://github.com/dotnet/razor/issues/7505")] @@ -531,7 +531,7 @@ public void HtmlTransition_05(bool withComment) var token = tree.Root.FindToken(position); - AssertEx.Equal("""OpenAngle;[<];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""OpenAngle;[<];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -546,7 +546,7 @@ public void IgnoreWhitespace_BeforeNewline_01() var token = tree.Root.FindToken(position); - AssertEx.Equal("""CloseAngle;[>];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""CloseAngle;[>];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -560,7 +560,7 @@ public void IgnoreWhitespace_BeforeNewline_02() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Text;[asdf];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Text;[asdf];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -575,7 +575,7 @@ public void IgnoreWhitespace_AfterNewline() var token = tree.Root.FindToken(position); - AssertEx.Equal("""OpenAngle;[<];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""OpenAngle;[<];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -589,7 +589,7 @@ public void IgnoreWhitespace_InsideTag_01() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Text;[div];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Text;[div];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -603,7 +603,7 @@ public void IgnoreWhitespace_InsideTag_02() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Text;[Attribute];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Text;[Attribute];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -617,7 +617,7 @@ public void IgnoreWhitespace_InsideTag_03() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Equals;[=];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Equals;[=];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -632,7 +632,7 @@ public void IgnoreWhitespace_InsideTag_04() var token = tree.Root.FindToken(position); - AssertEx.Equal("""DoubleQuote;["];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""DoubleQuote;["];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -646,7 +646,7 @@ public void IgnoreWhitespace_InsideTag_05() var token = tree.Root.FindToken(position); - AssertEx.Equal("""DoubleQuote;["];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""DoubleQuote;["];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -660,7 +660,7 @@ public void IgnoreWhitespace_InsideTag_06() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Text;[value];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Text;[value];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -674,7 +674,7 @@ public void IgnoreWhitespace_InsideTag_07() var token = tree.Root.FindToken(position); - AssertEx.Equal("""DoubleQuote;["];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""DoubleQuote;["];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -688,7 +688,7 @@ public void IgnoreWhitespace_InsideTag_08() var token = tree.Root.FindToken(position); - AssertEx.Equal("""CloseAngle;[>];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""CloseAngle;[>];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -703,7 +703,7 @@ public void IgnoreWhitespace_InsideTag_09() var token = tree.Root.FindToken(position); - AssertEx.Equal("""CloseAngle;[>];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""CloseAngle;[>];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -717,7 +717,7 @@ public void HtmlComment_01() var token = tree.Root.FindToken(position); - AssertEx.Equal("""OpenAngle;[<];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""OpenAngle;[<];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -731,7 +731,7 @@ public void HtmlComment_02() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Bang;[!];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Bang;[!];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -745,7 +745,7 @@ public void HtmlComment_03() var token = tree.Root.FindToken(position); - AssertEx.Equal("""DoubleHyphen;[--];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""DoubleHyphen;[--];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -759,7 +759,7 @@ public void HtmlComment_04_IgnoreWhitespace() var token = tree.Root.FindToken(position); - AssertEx.Equal("""DoubleHyphen;[--];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""DoubleHyphen;[--];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -773,7 +773,7 @@ public void HtmlComment_04_IncludeWhitespace() var token = tree.Root.FindToken(position, includeWhitespace: true); - AssertEx.Equal("""Whitespace;[ ];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Whitespace;[ ];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -787,7 +787,7 @@ public void HtmlComment_05() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Text;[Comment];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Text;[Comment];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -801,7 +801,7 @@ public void HtmlComment_06() var token = tree.Root.FindToken(position); - AssertEx.Equal("""Text;[Comment];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""Text;[Comment];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -815,7 +815,7 @@ public void HtmlComment_07() var token = tree.Root.FindToken(position); - AssertEx.Equal("""DoubleHyphen;[--];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""DoubleHyphen;[--];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -829,7 +829,7 @@ public void HtmlComment_08() var token = tree.Root.FindToken(position); - AssertEx.Equal("""CloseAngle;[>];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""CloseAngle;[>];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -843,7 +843,7 @@ public void HtmlComment_09() var token = tree.Root.FindToken(position); - AssertEx.Equal("""CloseAngle;[>];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""CloseAngle;[>];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -857,7 +857,7 @@ public void RazorComment_01() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RazorCommentTransition;[@];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RazorCommentTransition;[@];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -871,7 +871,7 @@ public void RazorComment_02() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RazorCommentStar;[*];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RazorCommentStar;[*];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -885,7 +885,7 @@ public void RazorComment_03_IgnoreWhitespace() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RazorCommentLiteral;[ Comment ];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RazorCommentLiteral;[ Comment ];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -899,7 +899,7 @@ public void RazorComment_03_IncludeWhitespace() var token = tree.Root.FindToken(position, includeWhitespace: true); - AssertEx.Equal("""RazorCommentLiteral;[ Comment ];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RazorCommentLiteral;[ Comment ];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -913,7 +913,7 @@ public void RazorComment_04() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RazorCommentLiteral;[ Comment ];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RazorCommentLiteral;[ Comment ];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -927,7 +927,7 @@ public void RazorComment_05() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RazorCommentLiteral;[ Comment ];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RazorCommentLiteral;[ Comment ];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -941,7 +941,7 @@ public void RazorComment_06() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RazorCommentStar;[*];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RazorCommentStar;[*];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -955,7 +955,7 @@ public void RazorComment_07() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RazorCommentTransition;[@];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RazorCommentTransition;[@];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] @@ -969,7 +969,7 @@ public void RazorComment_08() var token = tree.Root.FindToken(position); - AssertEx.Equal("""RazorCommentTransition;[@];""", SyntaxSerializer.Serialize(token).Trim()); + AssertEx.Equal("""RazorCommentTransition;[@];""", TestSyntaxSerializer.Serialize(token).Trim()); } [Fact] diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/DefaultDirectiveSyntaxTreePass.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/DefaultDirectiveSyntaxTreePass.cs index 80bed8804d6..8bf57143dd1 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/DefaultDirectiveSyntaxTreePass.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/DefaultDirectiveSyntaxTreePass.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; using Microsoft.AspNetCore.Razor.Language.Extensions; using Microsoft.AspNetCore.Razor.Language.Syntax; @@ -39,7 +40,8 @@ public RazorSyntaxTree Verify() return new RazorSyntaxTree(root, _syntaxTree.Source, diagnostics, _syntaxTree.Options); } - public override SyntaxNode Visit(SyntaxNode node) + [return: NotNullIfNotNull(nameof(node))] + public override SyntaxNode? Visit(SyntaxNode? node) { try { diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/DefaultRazorTagHelperContextDiscoveryPhase.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/DefaultRazorTagHelperContextDiscoveryPhase.cs index e349364cd1f..2a57f87eeb4 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/DefaultRazorTagHelperContextDiscoveryPhase.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/DefaultRazorTagHelperContextDiscoveryPhase.cs @@ -58,18 +58,18 @@ protected override void ExecuteCore(RazorCodeDocument codeDocument, Cancellation codeDocument.SetPreTagHelperSyntaxTree(syntaxTree); } - private static ReadOnlySpan GetSpanWithoutGlobalPrefix(string s) + internal static ReadOnlyMemory GetMemoryWithoutGlobalPrefix(string s) { const string globalPrefix = "global::"; - var span = s.AsSpan(); + var mem = s.AsMemory(); - if (span.StartsWith(globalPrefix.AsSpan(), StringComparison.Ordinal)) + if (mem.Span.StartsWith(globalPrefix.AsSpan(), StringComparison.Ordinal)) { - return span[globalPrefix.Length..]; + return mem[globalPrefix.Length..]; } - return span; + return mem; } internal abstract class DirectiveVisitor : SyntaxWalker @@ -241,7 +241,7 @@ public override void VisitRazorDirective(RazorDirectiveSyntax node) continue; } - switch (GetSpanWithoutGlobalPrefix(addTagHelper.TypePattern)) + switch (GetMemoryWithoutGlobalPrefix(addTagHelper.TypePattern).Span) { case ['*']: AddMatches(nonComponentTagHelpers); @@ -287,7 +287,7 @@ public override void VisitRazorDirective(RazorDirectiveSyntax node) continue; } - switch (GetSpanWithoutGlobalPrefix(removeTagHelper.TypePattern)) + switch (GetMemoryWithoutGlobalPrefix(removeTagHelper.TypePattern).Span) { case ['*']: RemoveMatches(nonComponentTagHelpers); @@ -334,7 +334,9 @@ public override void VisitRazorDirective(RazorDirectiveSyntax node) internal sealed class ComponentDirectiveVisitor : DirectiveVisitor { - private readonly List _nonFullyQualifiedComponents = []; + // The list values in this dictionary are pooled + private readonly Dictionary, List> _typeNamespaceToNonFullyQualifiedComponents = new Dictionary, List>(ReadOnlyMemoryOfCharComparer.Instance); + private List? _nonFullyQualifiedComponentsWithEmptyTypeNamespace; private string? _filePath; private RazorSourceDocument? _source; @@ -347,6 +349,7 @@ public void Initialize(string filePath, IReadOnlyList tagHe Debug.Assert(!IsInitialized); _filePath = filePath; + _nonFullyQualifiedComponentsWithEmptyTypeNamespace = ListPool.Default.Get(); foreach (var tagHelper in tagHelpers.AsEnumerable()) { @@ -363,25 +366,31 @@ public void Initialize(string filePath, IReadOnlyList tagHe continue; } - _nonFullyQualifiedComponents.Add(tagHelper); + var tagHelperTypeNamespace = tagHelper.GetTypeNamespace().AsMemory(); - if (currentNamespace is null) + if (tagHelperTypeNamespace.IsEmpty) { - continue; + _nonFullyQualifiedComponentsWithEmptyTypeNamespace.Add(tagHelper); } - - if (tagHelper.IsChildContentTagHelper) + else { - // If this is a child content tag helper, we want to add it if it's original type is in scope. - // E.g, if the type name is `Test.MyComponent.ChildContent`, we want to add it if `Test.MyComponent` is in scope. - if (IsTypeInScope(tagHelper, currentNamespace)) + if (!_typeNamespaceToNonFullyQualifiedComponents.TryGetValue(tagHelperTypeNamespace, out var tagHelpersList)) { - AddMatch(tagHelper); + tagHelpersList = ListPool.Default.Get(); + _typeNamespaceToNonFullyQualifiedComponents.Add(tagHelperTypeNamespace, tagHelpersList); } + + tagHelpersList.Add(tagHelper); } - else if (IsTypeInScope(tagHelper, currentNamespace)) + + if (currentNamespace is null) { - // Also, if the type is already in scope of the document's namespace, using isn't necessary. + continue; + } + + if (IsTypeNamespaceInScope(tagHelperTypeNamespace.Span, currentNamespace)) + { + // If the type is already in scope of the document's namespace, using isn't necessary. AddMatch(tagHelper); } } @@ -391,7 +400,18 @@ public void Initialize(string filePath, IReadOnlyList tagHe public override void Reset() { - _nonFullyQualifiedComponents.Clear(); + if (_nonFullyQualifiedComponentsWithEmptyTypeNamespace != null) + { + ListPool.Default.Return(_nonFullyQualifiedComponentsWithEmptyTypeNamespace); + _nonFullyQualifiedComponentsWithEmptyTypeNamespace = null; + } + + foreach (var (_, tagHelpers) in _typeNamespaceToNonFullyQualifiedComponents) + { + ListPool.Default.Return(tagHelpers); + } + + _typeNamespaceToNonFullyQualifiedComponents.Clear(); _filePath = null; _source = null; @@ -407,6 +427,8 @@ public override void Visit(RazorSyntaxTree tree) public override void VisitRazorDirective(RazorDirectiveSyntax node) { + var componentsWithEmptyTypeNamespace = _nonFullyQualifiedComponentsWithEmptyTypeNamespace.AssumeNotNull(); + var descendantLiterals = node.DescendantNodes(); foreach (var child in descendantLiterals) { @@ -455,22 +477,30 @@ public override void VisitRazorDirective(RazorDirectiveSyntax node) continue; } - foreach (var tagHelper in _nonFullyQualifiedComponents) + if (_typeNamespaceToNonFullyQualifiedComponents.Count == 0 && componentsWithEmptyTypeNamespace.Count == 0) + { + // There aren't any non qualified components to add + continue; + } + + // Add all tag helpers that have an empty type namespace + foreach (var tagHelper in componentsWithEmptyTypeNamespace) { Debug.Assert(!tagHelper.IsComponentFullyQualifiedNameMatch, "We've already processed these."); - if (tagHelper.IsChildContentTagHelper) - { - // If this is a child content tag helper, we want to add it if it's original type is in scope of the given namespace. - // E.g, if the type name is `Test.MyComponent.ChildContent`, we want to add it if `Test.MyComponent` is in this namespace. - if (IsTypeInNamespace(tagHelper, @namespace)) - { - AddMatch(tagHelper); - } - } - else if (IsTypeInNamespace(tagHelper, @namespace)) + AddMatch(tagHelper); + } + + // Remove global:: prefix from namespace. + var normalizedNamespace = GetMemoryWithoutGlobalPrefix(@namespace); + + // Add all tag helpers with a matching namespace + if (_typeNamespaceToNonFullyQualifiedComponents.TryGetValue(normalizedNamespace, out var tagHelpers)) + { + foreach (var tagHelper in tagHelpers) { - // If the type is at the top-level or if the type's namespace matches the using's namespace, add it. + Debug.Assert(!tagHelper.IsComponentFullyQualifiedNameMatch, "We've already processed these."); + AddMatch(tagHelper); } } @@ -480,32 +510,14 @@ public override void VisitRazorDirective(RazorDirectiveSyntax node) } } - internal static bool IsTypeInNamespace(TagHelperDescriptor tagHelper, string @namespace) - { - var typeNamespace = tagHelper.GetTypeNamespace(); - - if (typeNamespace.IsNullOrEmpty()) - { - // Either the typeName is not the full type name or this type is at the top level. - return true; - } - - // Remove global:: prefix from namespace. - var normalizedNamespaceSpan = GetSpanWithoutGlobalPrefix(@namespace); - - return normalizedNamespaceSpan.Equals(typeNamespace.AsSpan(), StringComparison.Ordinal); - } - - // Check if the given type is already in scope given the namespace of the current document. + // Check if a type's namespace is already in scope given the namespace of the current document. // E.g, // If the namespace of the document is `MyComponents.Components.Shared`, // then the types `MyComponents.FooComponent`, `MyComponents.Components.BarComponent`, `MyComponents.Components.Shared.BazComponent` are all in scope. // Whereas `MyComponents.SomethingElse.OtherComponent` is not in scope. - internal static bool IsTypeInScope(TagHelperDescriptor tagHelper, string @namespace) + internal static bool IsTypeNamespaceInScope(ReadOnlySpan typeNamespace, string @namespace) { - var typeNamespace = tagHelper.GetTypeNamespace(); - - if (typeNamespace.IsNullOrEmpty()) + if (typeNamespace.IsEmpty) { // Either the typeName is not the full type name or this type is at the top level. return true; diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/TagHelperBlockRewriter.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/TagHelperBlockRewriter.cs index 7731498b812..6464726bed6 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/TagHelperBlockRewriter.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/TagHelperBlockRewriter.cs @@ -621,7 +621,7 @@ public override SyntaxNode VisitCSharpImplicitExpression(CSharpImplicitExpressio var firstToken = firstChild.GetFirstToken(); var newFirstToken = SyntaxFactory.Token(firstToken.Kind, node.Transition.Transition.Content + firstToken.Content).WithAnnotations(firstToken.GetAnnotations()); - var newFirstChild = firstChild.ReplaceNode(firstToken, newFirstToken); + var newFirstChild = firstChild.ReplaceToken(firstToken, newFirstToken); builder.AddRange(rewrittenBody.Children.Replace(firstChild, newFirstChild)); // Since the original transition is part of the body, we need something to take it's place. diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/TagHelperParseTreeRewriter.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/TagHelperParseTreeRewriter.cs index dcc5167dd70..091ff3e91be 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/TagHelperParseTreeRewriter.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/TagHelperParseTreeRewriter.cs @@ -36,10 +36,7 @@ public static RazorSyntaxTree Rewrite(RazorSyntaxTree syntaxTree, TagHelperBinde foreach (var descriptor in binder.Descriptors) { - foreach (var diagnostic in descriptor.GetAllDiagnostics()) - { - builder.Add(diagnostic); - } + descriptor.AppendAllDiagnostics(ref builder.AsRef()); } var diagnostics = builder.ToImmutableOrderedBy(static d => d.Span.AbsoluteIndex); @@ -162,7 +159,7 @@ public override SyntaxNode VisitMarkupElement(MarkupElementSyntax node) var body = VisitList(node.Body); // Visit end tag. - var endTag = (MarkupEndTagSyntax)Visit(node.EndTag); + var endTag = (MarkupEndTagSyntax?)Visit(node.EndTag); if (endTag != null) { var tagName = endTag.GetTagNameWithOptionalBang(); diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/ReadOnlyMemoryOfCharComparer.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/ReadOnlyMemoryOfCharComparer.cs new file mode 100644 index 00000000000..802b2fd23bb --- /dev/null +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/ReadOnlyMemoryOfCharComparer.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Internal; + +namespace Microsoft.AspNetCore.Razor.Language; + +internal sealed class ReadOnlyMemoryOfCharComparer : IEqualityComparer> +{ + public static readonly ReadOnlyMemoryOfCharComparer Instance = new ReadOnlyMemoryOfCharComparer(); + + private ReadOnlyMemoryOfCharComparer() + { + } + + public static bool Equals(ReadOnlySpan x, ReadOnlyMemory y) + => x.SequenceEqual(y.Span); + + public bool Equals(ReadOnlyMemory x, ReadOnlyMemory y) + => x.Span.SequenceEqual(y.Span); + + public int GetHashCode(ReadOnlyMemory memory) + { +#if NET + return string.GetHashCode(memory.Span); +#else + // We don't rely on ReadOnlyMemory.GetHashCode() because it includes + // the index and length, but we just want a hash based on the characters. + var hashCombiner = HashCodeCombiner.Start(); + + foreach (var ch in memory.Span) + { + hashCombiner.Add(ch); + } + + return hashCombiner.CombinedHash; +#endif + } +} diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/RazorSyntaxNode.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/RazorSyntaxNode.cs index 9307de39f5d..fc8b07ca77a 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/RazorSyntaxNode.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/RazorSyntaxNode.cs @@ -1,7 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable disable +using System; +using System.Collections.Generic; namespace Microsoft.AspNetCore.Razor.Language.Syntax; @@ -11,4 +12,29 @@ public RazorSyntaxNode(GreenNode green, SyntaxNode parent, int position) : base(green, parent, position) { } + + internal override string SerializedValue => Serializer.Serialize(this); + + public abstract TResult? Accept(SyntaxVisitor visitor); + + public abstract void Accept(SyntaxVisitor visitor); + + protected internal override SyntaxNode ReplaceCore( + IEnumerable? nodes = null, + Func? computeReplacementNode = null, + IEnumerable? tokens = null, + Func? computeReplacementToken = null) + => SyntaxReplacer.Replace(this, nodes, computeReplacementNode, tokens, computeReplacementToken); + + protected internal override SyntaxNode ReplaceNodeInListCore(SyntaxNode originalNode, IEnumerable replacementNodes) + => SyntaxReplacer.ReplaceNodeInList(this, originalNode, replacementNodes); + + protected internal override SyntaxNode InsertNodesInListCore(SyntaxNode nodeInList, IEnumerable nodesToInsert, bool insertBefore) + => SyntaxReplacer.InsertNodeInList(this, nodeInList, nodesToInsert, insertBefore); + + protected internal override SyntaxNode ReplaceTokenInListCore(SyntaxToken originalToken, IEnumerable newTokens) + => SyntaxReplacer.ReplaceTokenInList(this, originalToken, newTokens); + + protected internal override SyntaxNode InsertTokensInListCore(SyntaxToken originalToken, IEnumerable newTokens, bool insertBefore) + => SyntaxReplacer.InsertTokenInList(this, originalToken, newTokens, insertBefore); } diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxList.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxList.cs index 678cdb9bd87..56d343db80f 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxList.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxList.cs @@ -1,6 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.Collections.Generic; + namespace Microsoft.AspNetCore.Razor.Language.Syntax; internal abstract class SyntaxList : SyntaxNode @@ -10,17 +13,28 @@ protected SyntaxList(InternalSyntax.SyntaxList green, SyntaxNode parent, int pos { } - public override TResult Accept(SyntaxVisitor visitor) - { - return visitor.Visit(this); - } + internal override string SerializedValue => $"List: {SlotCount} slots"; - public override void Accept(SyntaxVisitor visitor) - { - visitor.Visit(this); - } + protected internal override SyntaxNode ReplaceCore( + IEnumerable? nodes = null, + Func? computeReplacementNode = null, + IEnumerable? tokens = null, + Func? computeReplacementToken = null) + => Assumed.Unreachable(); + + protected internal override SyntaxNode ReplaceNodeInListCore(SyntaxNode originalNode, IEnumerable replacementNodes) + => Assumed.Unreachable(); + + protected internal override SyntaxNode InsertNodesInListCore(SyntaxNode nodeInList, IEnumerable nodesToInsert, bool insertBefore) + => Assumed.Unreachable(); + + protected internal override SyntaxNode ReplaceTokenInListCore(SyntaxToken originalToken, IEnumerable newTokens) + => Assumed.Unreachable(); + + protected internal override SyntaxNode InsertTokensInListCore(SyntaxToken originalToken, IEnumerable newTokens, bool insertBefore) + => Assumed.Unreachable(); - internal class WithTwoChildren : SyntaxList + internal sealed class WithTwoChildren : SyntaxList { private SyntaxNode? _child0; private SyntaxNode? _child1; @@ -47,7 +61,7 @@ internal WithTwoChildren(InternalSyntax.SyntaxList green, SyntaxNode parent, int }; } - internal class WithThreeChildren : SyntaxList + internal sealed class WithThreeChildren : SyntaxList { private SyntaxNode? _child0; private SyntaxNode? _child1; @@ -77,7 +91,7 @@ internal WithThreeChildren(InternalSyntax.SyntaxList green, SyntaxNode parent, i }; } - internal class WithManyChildren : SyntaxList + internal sealed class WithManyChildren : SyntaxList { private readonly ArrayElement[] _children; diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNode.Serializer.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNode.Serializer.cs new file mode 100644 index 00000000000..bf0cf90152f --- /dev/null +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNode.Serializer.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using Microsoft.AspNetCore.Razor.PooledObjects; + +namespace Microsoft.AspNetCore.Razor.Language.Syntax; + +internal abstract partial class SyntaxNode +{ + protected sealed class Serializer : SyntaxSerializer + { + private Serializer(StringBuilder builder) + : base(builder) + { + } + + internal static string Serialize(RazorSyntaxNode node) + { + using var _ = StringBuilderPool.GetPooledObject(out var builder); + var serializer = new Serializer(builder); + serializer.Visit(node); + + return builder.ToString(); + } + + internal static string Serialize(SyntaxToken token) + { + using var _ = StringBuilderPool.GetPooledObject(out var builder); + var serializer = new Serializer(builder); + serializer.VisitToken(token); + + return builder.ToString(); + } + } +} diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNode.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNode.cs index 7fbcf0b4581..018e7808dd3 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNode.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNode.cs @@ -42,11 +42,7 @@ internal abstract partial class SyntaxNode(GreenNode green, SyntaxNode parent, i public bool ContainsAnnotations => Green.ContainsAnnotations; - internal string SerializedValue => SyntaxSerializer.Serialize(this); - - public abstract TResult Accept(SyntaxVisitor visitor); - - public abstract void Accept(SyntaxVisitor visitor); + internal abstract string SerializedValue { get; } internal abstract SyntaxNode? GetNodeSlot(int index); @@ -280,23 +276,17 @@ public IEnumerable DescendantTokens(Func? descend return DescendantNodesImpl(Span, descendIntoChildren, includeSelf: true).OfType(); } - protected internal SyntaxNode ReplaceCore( + protected internal abstract SyntaxNode ReplaceCore( IEnumerable? nodes = null, - Func? computeReplacementNode = null) - where TNode : SyntaxNode - { - return SyntaxReplacer.Replace(this, nodes, computeReplacementNode); - } - - protected internal SyntaxNode ReplaceNodeInListCore(SyntaxNode originalNode, IEnumerable replacementNodes) - { - return SyntaxReplacer.ReplaceNodeInList(this, originalNode, replacementNodes); - } - - protected internal SyntaxNode InsertNodesInListCore(SyntaxNode nodeInList, IEnumerable nodesToInsert, bool insertBefore) - { - return SyntaxReplacer.InsertNodeInList(this, nodeInList, nodesToInsert, insertBefore); - } + Func? computeReplacementNode = null, + IEnumerable? tokens = null, + Func? computeReplacementToken = null) + where TNode : SyntaxNode; + + protected internal abstract SyntaxNode ReplaceNodeInListCore(SyntaxNode originalNode, IEnumerable replacementNodes); + protected internal abstract SyntaxNode InsertNodesInListCore(SyntaxNode nodeInList, IEnumerable nodesToInsert, bool insertBefore); + protected internal abstract SyntaxNode ReplaceTokenInListCore(SyntaxToken originalToken, IEnumerable newTokens); + protected internal abstract SyntaxNode InsertTokensInListCore(SyntaxToken originalToken, IEnumerable newTokens, bool insertBefore); public RazorDiagnostic[] GetDiagnostics() { diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNodeExtensions.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNodeExtensions.cs index 2aa9d639f9d..098305ef3a0 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNodeExtensions.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxNodeExtensions.cs @@ -120,11 +120,14 @@ public static SourceSpan GetSourceSpan(this SyntaxNode node, RazorSourceDocument public static TRoot ReplaceSyntax( this TRoot root, IEnumerable nodes, - Func computeReplacementNode) + Func computeReplacementNode, + IEnumerable tokens, + Func computeReplacementToken) where TRoot : SyntaxNode { return (TRoot)root.ReplaceCore( - nodes: nodes, computeReplacementNode: computeReplacementNode); + nodes: nodes, computeReplacementNode: computeReplacementNode, + tokens: tokens, computeReplacementToken: computeReplacementToken); } /// @@ -175,6 +178,35 @@ public static TRoot ReplaceNode(this TRoot root, SyntaxNode oldNode, IEnu return (TRoot)root.ReplaceNodeInListCore(oldNode, newNodes); } + /// + /// Creates a new tree of nodes with the specified old node replaced with a new node. + /// + /// The type of the root node. + /// The root node of the tree of nodes. + /// The token to be replaced; descendants of the root node. + /// A function that computes a replacement token for + /// the argument tokens. The first argument is the original token. The second argument is + /// the same token potentially rewritten with replaced trivia. + public static TRoot ReplaceTokens(this TRoot root, IEnumerable tokens, Func computeReplacementToken) + where TRoot : SyntaxNode + { + return (TRoot)root.ReplaceCore(tokens: tokens, computeReplacementToken: computeReplacementToken); + } + + /// + /// Creates a new tree of nodes with the specified old token replaced with a new token. + /// + /// The type of the root node. + /// The root node of the tree of nodes. + /// The token to be replaced. + /// The new token to use in the new tree in place of the old + /// token. + public static TRoot ReplaceToken(this TRoot root, SyntaxToken oldToken, SyntaxToken newToken) + where TRoot : SyntaxNode + { + return (TRoot)root.ReplaceCore(tokens: [oldToken], computeReplacementToken: (o, r) => newToken); + } + /// /// Creates a new tree of nodes with new nodes inserted before the specified node. /// @@ -225,5 +257,17 @@ public override void Visit(SyntaxNode node) base.Visit(node); } } + + public override void VisitToken(SyntaxToken token) + { + if (token.ContainsDiagnostics == true) + { + var diagnostics = token.GetDiagnostics(); + + _diagnostics.AddRange(diagnostics); + + base.VisitToken(token); + } + } } } diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxReplacer.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxReplacer.cs index 20b2304c9df..e6ce05481cf 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxReplacer.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxReplacer.cs @@ -1,10 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable disable - using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using Microsoft.CodeAnalysis.Text; @@ -14,11 +13,13 @@ internal static class SyntaxReplacer { internal static SyntaxNode Replace( SyntaxNode root, - IEnumerable nodes = null, - Func computeReplacementNode = null) + IEnumerable? nodes = null, + Func? computeReplacementNode = null, + IEnumerable? tokens = null, + Func? computeReplacementToken = null) where TNode : SyntaxNode { - var replacer = new Replacer(nodes, computeReplacementNode); + var replacer = new Replacer(nodes, computeReplacementNode, tokens, computeReplacementToken); if (replacer.HasWork) { @@ -30,60 +31,75 @@ internal static SyntaxNode Replace( } } - internal static SyntaxNode ReplaceNodeInList(SyntaxNode root, SyntaxNode originalNode, IEnumerable newNodes) + internal static SyntaxToken Replace( + SyntaxToken root, + IEnumerable? nodes = null, + Func? computeReplacementNode = null, + IEnumerable? tokens = null, + Func? computeReplacementToken = null) { - return new NodeListEditor(originalNode, newNodes, ListEditKind.Replace).Visit(root); - } + var replacer = new Replacer(nodes, computeReplacementNode, tokens, computeReplacementToken); - internal static SyntaxNode InsertNodeInList(SyntaxNode root, SyntaxNode nodeInList, IEnumerable nodesToInsert, bool insertBefore) - { - return new NodeListEditor(nodeInList, nodesToInsert, insertBefore ? ListEditKind.InsertBefore : ListEditKind.InsertAfter).Visit(root); + if (replacer.HasWork) + { + return replacer.VisitToken(root); + } + else + { + return root; + } } - private class Replacer : SyntaxRewriter where TNode : SyntaxNode + private sealed class Replacer : SyntaxRewriter where TNode : SyntaxNode { - private readonly Func _computeReplacementNode; + private static readonly HashSet s_noNodes = []; + private static readonly HashSet s_noTokens = []; + + private readonly Func? _computeReplacementNode; + private readonly Func? _computeReplacementToken; + private readonly HashSet _nodeSet; + private readonly HashSet _tokenSet; private readonly HashSet _spanSet; - private readonly TextSpan _totalSpan; - public Replacer(IEnumerable nodes, Func computeReplacementNode) + private TextSpan _totalSpan; + + public Replacer( + IEnumerable? nodes = null, + Func? computeReplacementNode = null, + IEnumerable? tokens = null, + Func? computeReplacementToken = null) { _computeReplacementNode = computeReplacementNode; - _nodeSet = nodes != null ? new HashSet(nodes) : new HashSet(); - _spanSet = new HashSet(_nodeSet.Select(n => n.Span)); - _totalSpan = ComputeTotalSpan(_spanSet); + _computeReplacementToken = computeReplacementToken; + + _nodeSet = nodes != null ? [.. nodes] : s_noNodes; + _tokenSet = tokens != null ? [.. tokens] : s_noTokens; + + _spanSet = []; + CalculateVisitationCriteria(); } - public bool HasWork => _nodeSet.Count > 0; + public bool HasWork => _nodeSet.Count + _tokenSet.Count > 0; - public override SyntaxNode Visit(SyntaxNode node) + private void CalculateVisitationCriteria() { - var rewritten = node; - - if (node != null) + _spanSet.Clear(); + foreach (var node in _nodeSet) { - if (ShouldVisit(node.Span)) - { - rewritten = base.Visit(node); - } - - if (_nodeSet.Contains(node) && _computeReplacementNode != null) - { - rewritten = _computeReplacementNode((TNode)node, (TNode)rewritten); - } + _spanSet.Add(node.Span); } - return rewritten; - } + foreach (var token in _tokenSet) + { + _spanSet.Add(token.Span); + } - private static TextSpan ComputeTotalSpan(IEnumerable spans) - { var first = true; var start = 0; var end = 0; - foreach (var span in spans) + foreach (var span in _spanSet) { if (first) { @@ -98,7 +114,7 @@ private static TextSpan ComputeTotalSpan(IEnumerable spans) } } - return new TextSpan(start, end - start); + _totalSpan = new TextSpan(start, end - start); } private bool ShouldVisit(TextSpan span) @@ -123,24 +139,101 @@ private bool ShouldVisit(TextSpan span) return false; } + + [return: NotNullIfNotNull(nameof(node))] + public override SyntaxNode? Visit(SyntaxNode? node) + { + var rewritten = node; + + if (node != null) + { + var isReplacedNode = _nodeSet.Remove(node); + + if (isReplacedNode) + { + // If node is in _nodeSet, then it contributed to the calculation of _spanSet. + // We are currently processing that node, so it no longer needs to contribute + // to _spanSet and affect determination of inward visitation. This is done before + // calling ShouldVisit to avoid walking into the node if there aren't any remaining + // spans inside it representing items to replace. + CalculateVisitationCriteria(); + } + + if (ShouldVisit(node.Span)) + { + rewritten = base.Visit(node); + } + + if (isReplacedNode && _computeReplacementNode != null) + { + rewritten = _computeReplacementNode((TNode)node, (TNode)rewritten!); + } + } + + return rewritten; + } + + public override SyntaxToken VisitToken(SyntaxToken token) + { + var rewritten = token; + var isReplacedToken = _tokenSet.Remove(token); + + if (isReplacedToken) + { + // If token is in _tokenSet, then it contributed to the calculation of _spanSet. + // We are currently processing that token, so it no longer needs to contribute + // to _spanSet and affect determination of inward visitation. This is done before + // calling ShouldVisit to avoid walking into the token if there aren't any remaining + // spans inside it representing items to replace. + CalculateVisitationCriteria(); + } + + if (isReplacedToken && _computeReplacementToken != null) + { + rewritten = _computeReplacementToken(token, rewritten); + } + + return rewritten; + } + } + + internal static SyntaxNode ReplaceNodeInList(SyntaxNode root, SyntaxNode originalNode, IEnumerable newNodes) + { + return new NodeListEditor(originalNode, newNodes, ListEditKind.Replace).Visit(root); + } + + internal static SyntaxNode InsertNodeInList(SyntaxNode root, SyntaxNode nodeInList, IEnumerable nodesToInsert, bool insertBefore) + { + return new NodeListEditor(nodeInList, nodesToInsert, insertBefore ? ListEditKind.InsertBefore : ListEditKind.InsertAfter).Visit(root); } - private class NodeListEditor : SyntaxRewriter + public static SyntaxNode ReplaceTokenInList(SyntaxNode root, SyntaxToken tokenInList, IEnumerable newTokens) + { + return new TokenListEditor(tokenInList, newTokens, ListEditKind.Replace).Visit(root); + } + + public static SyntaxNode InsertTokenInList(SyntaxNode root, SyntaxToken tokenInList, IEnumerable newTokens, bool insertBefore) + { + return new TokenListEditor(tokenInList, newTokens, insertBefore ? ListEditKind.InsertBefore : ListEditKind.InsertAfter).Visit(root); + } + + private enum ListEditKind + { + InsertBefore, + InsertAfter, + Replace + } + + private abstract class BaseListEditor : SyntaxRewriter { private readonly TextSpan _elementSpan; - private readonly SyntaxNode _originalNode; - private readonly IEnumerable _newNodes; - private readonly ListEditKind _editKind; - public NodeListEditor( - SyntaxNode originalNode, - IEnumerable replacementNodes, - ListEditKind editKind) + protected readonly ListEditKind EditKind; + + protected BaseListEditor(TextSpan elementSpan, ListEditKind editKind) { - _elementSpan = originalNode.Span; - _originalNode = originalNode; - _newNodes = replacementNodes; - _editKind = editKind; + _elementSpan = elementSpan; + EditKind = editKind; } private bool ShouldVisit(TextSpan span) @@ -155,14 +248,10 @@ private bool ShouldVisit(TextSpan span) return false; } - public override SyntaxNode Visit(SyntaxNode node) + [return: NotNullIfNotNull(nameof(node))] + public override SyntaxNode? Visit(SyntaxNode? node) { - if (node == _originalNode) - { - throw new InvalidOperationException("Expecting a list"); - } - - var rewritten = node; + SyntaxNode? rewritten = node; if (node != null) { @@ -174,6 +263,33 @@ public override SyntaxNode Visit(SyntaxNode node) return rewritten; } + } + + private sealed class NodeListEditor : BaseListEditor + { + private readonly SyntaxNode _originalNode; + private readonly IEnumerable _newNodes; + + public NodeListEditor( + SyntaxNode originalNode, + IEnumerable replacementNodes, + ListEditKind editKind) + : base(originalNode.Span, editKind) + { + _originalNode = originalNode; + _newNodes = replacementNodes; + } + + [return: NotNullIfNotNull(nameof(node))] + public override SyntaxNode? Visit(SyntaxNode? node) + { + if (node == _originalNode) + { + throw new InvalidOperationException("Expecting a list"); + } + + return base.Visit(node); + } public override SyntaxList VisitList(SyntaxList list) { @@ -182,7 +298,7 @@ public override SyntaxList VisitList(SyntaxList list) var index = list.IndexOf((TNode)_originalNode); if (index >= 0 && index < list.Count) { - switch (_editKind) + switch (EditKind) { case ListEditKind.Replace: return list.ReplaceRange((TNode)_originalNode, _newNodes.Cast()); @@ -196,14 +312,54 @@ public override SyntaxList VisitList(SyntaxList list) } } - return base.VisitList(list); + return base.VisitList(list); } } - private enum ListEditKind + private class TokenListEditor : BaseListEditor { - InsertBefore, - InsertAfter, - Replace + private readonly SyntaxToken _originalToken; + private readonly IEnumerable _newTokens; + + public TokenListEditor( + SyntaxToken originalToken, + IEnumerable newTokens, + ListEditKind editKind) + : base(originalToken.Span, editKind) + { + _originalToken = originalToken; + _newTokens = newTokens; + } + + public override SyntaxToken VisitToken(SyntaxToken token) + { + if (token == _originalToken) + { + throw new InvalidOperationException("Expecting a list"); + } + + return base.VisitToken(token); + } + + public override SyntaxList VisitList(SyntaxList list) + { + var index = list.IndexOf(_originalToken); + if (index >= 0 && index < list.Count) + { + switch (EditKind) + { + case ListEditKind.Replace: + return list.ReplaceRange(_originalToken, _newTokens); + + case ListEditKind.InsertAfter: + return list.InsertRange(index + 1, _newTokens); + + case ListEditKind.InsertBefore: + return list.InsertRange(index, _newTokens); + } + } + + return base.VisitList(list); + } } } diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxRewriter.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxRewriter.cs index 9ad1a77e468..7604f7902aa 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxRewriter.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxRewriter.cs @@ -1,8 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable disable - +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using Microsoft.AspNetCore.Razor.PooledObjects; namespace Microsoft.AspNetCore.Razor.Language.Syntax; @@ -11,17 +11,21 @@ internal abstract partial class SyntaxRewriter : SyntaxVisitor { private int _recursionDepth; - public override SyntaxNode Visit(SyntaxNode node) + [return: NotNullIfNotNull(nameof(node))] + public override SyntaxNode? Visit(SyntaxNode? node) { if (node != null) { + Debug.Assert(!node.IsToken); + Debug.Assert(!node.IsList); + _recursionDepth++; StackGuard.EnsureSufficientExecutionStack(_recursionDepth); - var result = node.Accept(this); + var result = ((RazorSyntaxNode)node).Accept(this); _recursionDepth--; - return result; + return result!; } else { @@ -29,8 +33,13 @@ public override SyntaxNode Visit(SyntaxNode node) } } + public virtual SyntaxToken VisitToken(SyntaxToken token) + { + return token; + } + public virtual SyntaxList VisitList(SyntaxList list) - where TNode : SyntaxNode + where TNode : RazorSyntaxNode { var count = list.Count; if (count == 0) @@ -70,13 +79,49 @@ public virtual SyntaxList VisitList(SyntaxList list) : list; } - public override SyntaxNode VisitToken(SyntaxToken token) + public virtual TNode? VisitListElement(TNode? node) + where TNode : RazorSyntaxNode { - return token; + return (TNode?)Visit(node); } - public virtual TNode VisitListElement(TNode node) where TNode : SyntaxNode + public virtual SyntaxList VisitList(SyntaxList list) { - return (TNode)(SyntaxNode)Visit(node); + var count = list.Count; + if (count == 0) + { + return list; + } + + using PooledArrayBuilder builder = []; + + var isUpdating = false; + + for (var i = 0; i < count; i++) + { + var item = list[i]; + + var visited = VisitToken(item); + + if (item != visited && !isUpdating) + { + // The list is being updated, so we need to initialize the builder + // add the items we've seen so far. + builder.SetCapacityIfLarger(count); + + builder.AddRange(list, index: 0, count: i); + + isUpdating = true; + } + + if (isUpdating && visited != null && visited.Kind != SyntaxKind.None) + { + builder.Add(visited); + } + } + + return isUpdating + ? builder.ToList() + : list; } } diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxSerializer.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxSerializer.cs index 531be95fc2a..290e094b080 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxSerializer.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxSerializer.cs @@ -1,323 +1,248 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable disable - using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Text; using Microsoft.AspNetCore.Razor.Language.Legacy; +using Microsoft.CodeAnalysis.Text; namespace Microsoft.AspNetCore.Razor.Language.Syntax; -internal class SyntaxSerializer +internal abstract class SyntaxSerializer(StringBuilder builder) : SyntaxWalker { - internal static string Serialize(SyntaxNode node) - { - using (var writer = new StringWriter()) - { - var walker = new Walker(writer); - walker.Visit(node); + protected const int IndentSize = 4; + protected const string Separator = " - "; - return writer.ToString(); - } - } + protected readonly StringBuilder Builder = builder; + private bool _visitedRoot; - private class Walker : SyntaxWalker - { - private readonly SyntaxWriter _visitor; - private readonly TextWriter _writer; + private int _depth; - public Walker(TextWriter writer) + public sealed override void Visit(SyntaxNode? node) + { + if (node == null) { - _visitor = new SyntaxWriter(writer); - _writer = writer; + return; } - public TextWriter Writer { get; } + WriteNode(node); + Builder.AppendLine(); - public override SyntaxNode Visit(SyntaxNode node) - { - if (node == null) - { - return node; - } - - if (node.IsList) - { - return base.DefaultVisit(node); - } - - _visitor.Visit(node); - _writer.WriteLine(); - - if (!node.IsToken) - { - _visitor.Depth++; - node = base.DefaultVisit(node); - _visitor.Depth--; - } - - return node; - } + IncreaseIndent(); + base.Visit(node); + DecreaseIndent(); } - private class SyntaxWalker : SyntaxRewriter + public sealed override void VisitToken(SyntaxToken token) { - private readonly List _ancestors = new List(); - - protected IReadOnlyList Ancestors => _ancestors; - - protected SyntaxNode Parent => _ancestors.Count > 0 ? _ancestors[0] : null; - - protected override SyntaxNode DefaultVisit(SyntaxNode node) - { - _ancestors.Insert(0, node); - - try - { - for (var i = 0; i < node.SlotCount; i++) - { - var child = node.GetNodeSlot(i); - Visit(child); - } - } - finally - { - _ancestors.RemoveAt(0); - } - - return node; - } + WriteToken(token); + Builder.AppendLine(); } - private class SyntaxWriter : SyntaxRewriter + private void WriteNode(SyntaxNode node) { - private readonly TextWriter _writer; - private bool _visitedRoot; - - public SyntaxWriter(TextWriter writer) - { - _writer = writer; - } - - public int Depth { get; set; } - - public override SyntaxNode Visit(SyntaxNode node) - { - if (node is SyntaxToken token) - { - return VisitToken(token); - } - - WriteNode(node); - return node; - } - - public override SyntaxNode VisitToken(SyntaxToken token) - { - WriteToken(token); - return base.VisitToken(token); - } + WriteIndent(); + WriteValue(node.Kind); + WriteSeparator(); + WriteSpan(node.Span); - private void WriteNode(SyntaxNode node) + switch (node) { - WriteIndent(); - Write(node.Kind); - WriteSeparator(); - Write($"[{node.Position}..{node.EndPosition})"); - WriteSeparator(); - Write($"Width: {node.Width}"); - - if (node is RazorDirectiveSyntax razorDirective) - { + case RazorDirectiveSyntax razorDirective: WriteRazorDirective(razorDirective); - } - else if (node is MarkupTagHelperElementSyntax tagHelperElement) - { + break; + + case MarkupTagHelperElementSyntax tagHelperElement: WriteTagHelperElement(tagHelperElement); - } - else if (node is MarkupTagHelperAttributeSyntax tagHelperAttribute) - { + break; + + case MarkupTagHelperAttributeSyntax tagHelperAttribute: WriteTagHelperAttributeInfo(tagHelperAttribute.TagHelperAttributeInfo); - } - else if (node is MarkupMinimizedTagHelperAttributeSyntax minimizedTagHelperAttribute) - { + break; + + case MarkupMinimizedTagHelperAttributeSyntax minimizedTagHelperAttribute: WriteTagHelperAttributeInfo(minimizedTagHelperAttribute.TagHelperAttributeInfo); - } - else if (node is MarkupStartTagSyntax startTag) - { + break; + + case MarkupStartTagSyntax startTag: if (startTag.IsMarkupTransition) { WriteSeparator(); - Write("MarkupTransition"); + WriteValue("MarkupTransition"); } - } - else if (node is MarkupEndTagSyntax endTag) - { + + break; + + case MarkupEndTagSyntax endTag: if (endTag.IsMarkupTransition) { WriteSeparator(); - Write("MarkupTransition"); + WriteValue("MarkupTransition"); } - } - - if (ShouldDisplayNodeContent(node)) - { - WriteSeparator(); - Write($"[{node.GetContent()}]"); - } - - WriteChunkGenerator(node); - - var annotation = node.GetAnnotations().FirstOrDefault(a => a.Kind == SyntaxConstants.EditHandlerKind); - if (annotation != null && annotation.Data is SpanEditHandler handler) - { - WriteEditHandler(handler); - } - - if (!_visitedRoot) - { - WriteSeparator(); - Write($"[{node}]"); - _visitedRoot = true; - } - } - - private void WriteRazorDirective(RazorDirectiveSyntax node) - { - if (node.DirectiveDescriptor == null) - { - return; - } - - var builder = new StringBuilder("Directive:{"); - builder.Append(node.DirectiveDescriptor.Directive); - builder.Append(';'); - builder.Append(node.DirectiveDescriptor.Kind); - builder.Append(';'); - builder.Append(node.DirectiveDescriptor.Usage); - builder.Append('}'); - - var diagnostics = node.GetDiagnostics(); - if (diagnostics.Length > 0) - { - builder.Append(" ["); - var ids = string.Join(", ", diagnostics.Select(diagnostic => $"{diagnostic.Id}{diagnostic.Span}")); - builder.Append(ids); - builder.Append(']'); - } - WriteSeparator(); - Write(builder.ToString()); + break; } - private void WriteTagHelperElement(MarkupTagHelperElementSyntax node) + if (ShouldDisplayNodeContent(node)) { - // Write tag name WriteSeparator(); - Write($"{node.TagHelperInfo.TagName}[{node.TagHelperInfo.TagMode}]"); + WriteValue($"[{node.GetContent()}]"); + } - // Write descriptors - foreach (var descriptor in node.TagHelperInfo.BindingResult.Descriptors) - { - WriteSeparator(); + WriteChunkGenerator(node); - // Get the type name without the namespace. - var typeName = descriptor.Name.Substring(descriptor.Name.LastIndexOf('.') + 1); - Write(typeName); - } - } + WriteSpanEditHandlers(node); - private void WriteTagHelperAttributeInfo(TagHelperAttributeInfo info) + if (!_visitedRoot) { - // Write attributes WriteSeparator(); - Write(info.Name); - WriteSeparator(); - Write(info.AttributeStructure); - WriteSeparator(); - Write(info.Bound ? "Bound" : "Unbound"); + WriteValue($"[{node}]"); + _visitedRoot = true; } + } + + protected virtual void WriteSpan(TextSpan span) + { + WriteValue($"[{span.Start}..{span.End}{Separator}Width: {span.End - span.Start}"); + } - private void WriteToken(SyntaxToken token) + private void WriteRazorDirective(RazorDirectiveSyntax node) + { + if (node.DirectiveDescriptor is not { } descriptor) { - WriteIndent(); - var content = token.IsMissing ? "" : token.Content; - var diagnostics = token.GetDiagnostics(); - var tokenString = $"{token.Kind};[{content}];{string.Join(", ", diagnostics.Select(diagnostic => diagnostic.Id + diagnostic.Span))}"; - Write(tokenString); + return; } - private void WriteChunkGenerator(SyntaxNode node) + WriteSeparator(); + WriteValue($"Directive:{{{descriptor.Directive};{descriptor.Kind};{descriptor.Usage}}}"); + + if (node.GetDiagnostics() is { Length: > 0} diagnostics) { - var generator = node.GetChunkGenerator(); - if (generator != null) - { - WriteSeparator(); - Write($"Gen<{generator}>"); - } + WriteValue($" [{GetDiagnosticsText(diagnostics)}]"); } + } - private void WriteEditHandler(SpanEditHandler handler) + private void WriteTagHelperElement(MarkupTagHelperElementSyntax node) + { + var tagHelperInfo = node.TagHelperInfo.AssumeNotNull(); + + // Write tag name + WriteSeparator(); + WriteValue($"{tagHelperInfo.TagName}[{tagHelperInfo.TagMode}]"); + + // Write descriptors + foreach (var descriptor in tagHelperInfo.BindingResult.Descriptors) { WriteSeparator(); - Write(handler); - } - protected void WriteIndent() - { - for (var i = 0; i < Depth; i++) - { - for (var j = 0; j < 4; j++) - { - Write(' '); - } - } + // Get the type name without the namespace. + var typeName = descriptor.Name[(descriptor.Name.LastIndexOf('.') + 1)..]; + WriteValue(typeName); } + } - protected void WriteSeparator() - { - Write(" - "); - } + private void WriteTagHelperAttributeInfo(TagHelperAttributeInfo info) + { + // Write attributes + WriteValue($"{Separator}{info.Name}{Separator}{info.AttributeStructure}{Separator}{(info.Bound ? "Bound" : "Unbound")}"); + } + + private void WriteToken(SyntaxToken token) + { + WriteIndent(); + + var content = token.IsMissing ? "" : token.Content; + var diagnostics = token.GetDiagnostics(); + var diagnosticsText = GetDiagnosticsText(diagnostics); + + WriteValue($"{token.Kind};[{content}];{diagnosticsText}"); + } + + private static string GetDiagnosticsText(RazorDiagnostic[] diagnostics) + { + return diagnostics.Length > 0 + ? string.Join(", ", diagnostics.Select(diagnostic => $"{diagnostic.Id}{diagnostic.Span}")) + : string.Empty; + } - protected void WriteNewLine() + private void WriteChunkGenerator(SyntaxNode node) + { + if (node.GetChunkGenerator() is { } generator) { - _writer.WriteLine(); + WriteSeparator(); + WriteValue($"Gen<{generator}>"); } + } - protected void Write(object value) + protected virtual void WriteSpanEditHandlers(SyntaxNode node) + { + var annotation = node.GetAnnotations().FirstOrDefault(a => a.Kind == SyntaxConstants.EditHandlerKind); + if (annotation?.Data is SpanEditHandler handler) { - if (value is string stringValue) - { - stringValue = stringValue.Replace(Environment.NewLine, "LF"); - _writer.Write(stringValue); - return; - } - - _writer.Write(value); + WriteSeparator(); + WriteValue(handler); } + } + + protected void IncreaseIndent() + { + _depth++; + } + + protected void DecreaseIndent() + { + Assumed.True(--_depth >= 0, "Depth can't be less than 0."); + } + + protected void WriteIndent() + { + WriteValue(new string(' ', _depth * IndentSize)); + } + + protected void WriteSeparator() + { + WriteValue(Separator); + } + + protected virtual void WriteValue(string value) + { + Builder.Append(value.Replace(Environment.NewLine, "LF")); + } - private static bool ShouldDisplayNodeContent(SyntaxNode node) + protected virtual void WriteValue(object? value) + { + switch (value) { - return node.Kind == SyntaxKind.MarkupTextLiteral || - node.Kind == SyntaxKind.MarkupEphemeralTextLiteral || - node.Kind == SyntaxKind.MarkupStartTag || - node.Kind == SyntaxKind.MarkupEndTag || - node.Kind == SyntaxKind.MarkupTagHelperStartTag || - node.Kind == SyntaxKind.MarkupTagHelperEndTag || - node.Kind == SyntaxKind.MarkupAttributeBlock || - node.Kind == SyntaxKind.MarkupMinimizedAttributeBlock || - node.Kind == SyntaxKind.MarkupTagHelperAttribute || - node.Kind == SyntaxKind.MarkupMinimizedTagHelperAttribute || - node.Kind == SyntaxKind.MarkupLiteralAttributeValue || - node.Kind == SyntaxKind.MarkupDynamicAttributeValue || - node.Kind == SyntaxKind.CSharpStatementLiteral || - node.Kind == SyntaxKind.CSharpExpressionLiteral || - node.Kind == SyntaxKind.CSharpEphemeralTextLiteral || - node.Kind == SyntaxKind.UnclassifiedTextLiteral; + case string s: + WriteValue(s); + break; + + default: + Builder.Append(value); + break; } } + + private static bool ShouldDisplayNodeContent(SyntaxNode node) + { + return node.Kind is + SyntaxKind.MarkupTextLiteral or + SyntaxKind.MarkupEphemeralTextLiteral or + SyntaxKind.MarkupStartTag or + SyntaxKind.MarkupEndTag or + SyntaxKind.MarkupTagHelperStartTag or + SyntaxKind.MarkupTagHelperEndTag or + SyntaxKind.MarkupAttributeBlock or + SyntaxKind.MarkupMinimizedAttributeBlock or + SyntaxKind.MarkupTagHelperAttribute or + SyntaxKind.MarkupMinimizedTagHelperAttribute or + SyntaxKind.MarkupLiteralAttributeValue or + SyntaxKind.MarkupDynamicAttributeValue or + SyntaxKind.CSharpStatementLiteral or + SyntaxKind.CSharpExpressionLiteral or + SyntaxKind.CSharpEphemeralTextLiteral or + SyntaxKind.UnclassifiedTextLiteral; + } } diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxToken.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxToken.cs index 27c62c0c0b4..001c940e137 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxToken.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxToken.cs @@ -1,13 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable disable - using System; +using System.Collections.Generic; namespace Microsoft.AspNetCore.Razor.Language.Syntax; -internal class SyntaxToken : RazorSyntaxNode +internal class SyntaxToken : SyntaxNode { internal static readonly Func NonZeroWidth = t => t.Width > 0; internal static readonly Func Any = t => true; @@ -17,6 +16,8 @@ internal SyntaxToken(GreenNode green, SyntaxNode parent, int position) { } + internal override string SerializedValue => Serializer.Serialize(this); + internal new InternalSyntax.SyntaxToken Green => (InternalSyntax.SyntaxToken)base.Green; public string Content => Green.Content; @@ -31,21 +32,30 @@ internal sealed override SyntaxNode GetNodeSlot(int slot) throw new InvalidOperationException("Tokens can't have slots."); } - public override TResult Accept(SyntaxVisitor visitor) - { - return visitor.VisitToken(this); - } + protected internal override SyntaxNode ReplaceCore( + IEnumerable? nodes = null, + Func? computeReplacementNode = null, + IEnumerable? tokens = null, + Func? computeReplacementToken = null) + => Assumed.Unreachable(); - public override void Accept(SyntaxVisitor visitor) - { - visitor.VisitToken(this); - } + protected internal override SyntaxNode ReplaceNodeInListCore(SyntaxNode originalNode, IEnumerable replacementNodes) + => Assumed.Unreachable(); + + protected internal override SyntaxNode InsertNodesInListCore(SyntaxNode nodeInList, IEnumerable nodesToInsert, bool insertBefore) + => Assumed.Unreachable(); + + protected internal override SyntaxNode ReplaceTokenInListCore(SyntaxToken originalToken, IEnumerable newTokens) + => Assumed.Unreachable(); + + protected internal override SyntaxNode InsertTokensInListCore(SyntaxToken originalToken, IEnumerable newTokens, bool insertBefore) + => Assumed.Unreachable(); /// /// Gets the token that follows this token in the syntax tree. /// /// The token that follows this token in the syntax tree. - public SyntaxToken GetNextToken(bool includeZeroWidth = false) + public SyntaxToken? GetNextToken(bool includeZeroWidth = false) { return SyntaxNavigator.GetNextToken(this, includeZeroWidth); } @@ -54,7 +64,7 @@ public SyntaxToken GetNextToken(bool includeZeroWidth = false) /// Gets the token that precedes this token in the syntax tree. /// /// The previous token that precedes this token in the syntax tree. - public SyntaxToken GetPreviousToken(bool includeZeroWidth = false) + public SyntaxToken? GetPreviousToken(bool includeZeroWidth = false) { return SyntaxNavigator.GetPreviousToken(this, includeZeroWidth); } diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxVisitor.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxVisitor.cs index 5433eea93a0..82bd15fced4 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxVisitor.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxVisitor.cs @@ -1,61 +1,62 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable disable +using System.Diagnostics; namespace Microsoft.AspNetCore.Razor.Language.Syntax; /// -/// Represents a visitor that visits only the single SyntaxNode -/// passed into its Visit method and produces -/// a value of the type specified by the parameter. +/// Represents a visitor that visits only the single +/// RazorSyntaxNode passed into its Visit method and produces a value of the type +/// specified by the parameter. /// /// -/// The type of the return value this visitor's Visit method. +/// The type of the return value this visitor's Visit method. /// internal abstract partial class SyntaxVisitor { - public virtual TResult Visit(SyntaxNode node) + public virtual TResult? Visit(SyntaxNode? node) { if (node != null) { - return node.Accept(this); + Debug.Assert(!node.IsToken); + Debug.Assert(!node.IsList); + + return ((RazorSyntaxNode)node).Accept(this); } - return default(TResult); + return default; } - public virtual TResult VisitToken(SyntaxToken token) + protected virtual TResult? DefaultVisit(SyntaxNode node) { - return DefaultVisit(token); - } + Debug.Assert(!node.IsToken); + Debug.Assert(!node.IsList); - protected virtual TResult DefaultVisit(SyntaxNode node) - { - return default(TResult); + return default; } } /// -/// Represents a visitor that visits only the single SyntaxNode -/// passed into its Visit method. +/// Represents a visitor that visits only the single +/// RazorSyntaxNode passed into its Visit method. /// internal abstract partial class SyntaxVisitor { - public virtual void Visit(SyntaxNode node) + public virtual void Visit(SyntaxNode? node) { if (node != null) { - node.Accept(this); - } - } + Debug.Assert(!node.IsToken); + Debug.Assert(!node.IsList); - public virtual void VisitToken(SyntaxToken token) - { - DefaultVisit(token); + ((RazorSyntaxNode)node).Accept(this); + } } public virtual void DefaultVisit(SyntaxNode node) { + Debug.Assert(!node.IsToken); + Debug.Assert(!node.IsList); } } diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxWalker.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxWalker.cs index 759905aef46..2993754640c 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxWalker.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Syntax/SyntaxWalker.cs @@ -1,9 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable disable - -using Microsoft.CodeAnalysis.Text; +using System.Diagnostics; namespace Microsoft.AspNetCore.Razor.Language.Syntax; @@ -16,21 +14,18 @@ namespace Microsoft.AspNetCore.Razor.Language.Syntax; internal abstract class SyntaxWalker : SyntaxVisitor { private int _recursionDepth; - private readonly TextSpan? _range; - - public SyntaxWalker(TextSpan? range = null) - { - _range = range; - } - public override void Visit(SyntaxNode node) + public override void Visit(SyntaxNode? node) { - if (node != null && (_range is null || _range.Value.OverlapsWith(node.Span))) + if (node != null) { + Debug.Assert(!node.IsToken); + Debug.Assert(!node.IsList); + _recursionDepth++; StackGuard.EnsureSufficientExecutionStack(_recursionDepth); - node.Accept(this); + ((RazorSyntaxNode)node).Accept(this); _recursionDepth--; } @@ -38,15 +33,23 @@ public override void Visit(SyntaxNode node) public override void DefaultVisit(SyntaxNode node) { - var children = node.ChildNodes(); - for (var i = 0; i < children.Count; i++) - { - var child = children[i]; + Debug.Assert(!node.IsToken); + Debug.Assert(!node.IsList); - if (_range is null || _range.Value.OverlapsWith(node.Span)) + foreach (var child in node.ChildNodes()) + { + if (child is SyntaxToken token) + { + VisitToken(token); + } + else { Visit(child); } } } + + public virtual void VisitToken(SyntaxToken token) + { + } } diff --git a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/TagHelperDescriptor.cs b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/TagHelperDescriptor.cs index 6ebaf55c66d..dd290b221df 100644 --- a/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/TagHelperDescriptor.cs +++ b/src/razor/src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/TagHelperDescriptor.cs @@ -173,35 +173,35 @@ static ImmutableArray GetEditorRequiredAttributes(Immu } public IEnumerable GetAllDiagnostics() + { + using var diagnostics = new PooledArrayBuilder(); + + AppendAllDiagnostics(ref diagnostics.AsRef()); + + foreach (var diagnostic in diagnostics) + { + yield return diagnostic; + } + } + + internal void AppendAllDiagnostics(ref PooledArrayBuilder diagnostics) { foreach (var allowedChildTag in AllowedChildTags) { - foreach (var diagnostic in allowedChildTag.Diagnostics) - { - yield return diagnostic; - } + diagnostics.AddRange(allowedChildTag.Diagnostics); } foreach (var boundAttribute in BoundAttributes) { - foreach (var diagnostic in boundAttribute.Diagnostics) - { - yield return diagnostic; - } + diagnostics.AddRange(boundAttribute.Diagnostics); } foreach (var tagMatchingRule in TagMatchingRules) { - foreach (var diagnostic in tagMatchingRule.Diagnostics) - { - yield return diagnostic; - } + diagnostics.AddRange(tagMatchingRule.Diagnostics); } - foreach (var diagnostic in Diagnostics) - { - yield return diagnostic; - } + diagnostics.AddRange(Diagnostics); } public override string ToString() diff --git a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Completion/AbstractRazorCompletionFactsService.cs b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Completion/AbstractRazorCompletionFactsService.cs index d1c4924caea..f01f2eba19f 100644 --- a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Completion/AbstractRazorCompletionFactsService.cs +++ b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Completion/AbstractRazorCompletionFactsService.cs @@ -5,6 +5,7 @@ using System.Collections.Immutable; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using Microsoft.AspNetCore.Razor.Language.Syntax; using Microsoft.AspNetCore.Razor.PooledObjects; using Microsoft.VisualStudio.Editor.Razor; @@ -78,6 +79,16 @@ public ImmutableArray GetCompletionItems(RazorCompletionCon return previousToken2.Parent; } + // If we have @ transition right in front of an existing equals and caret is after @, e.g. + // + // we get entire attribute from FindInnermostNode. We always want the attribute name as the context in such cases, + // so we adjust it to be the attrbute name node. + if (originalNode is MarkupAttributeBlockSyntax markupAttribute + && markupAttribute.EqualsToken.SpanStart == requestIndex) + { + return markupAttribute.Name; + } + return originalNode; } } diff --git a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Completion/MarkupTransitionCompletionItemProvider.cs b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Completion/MarkupTransitionCompletionItemProvider.cs index 4278e2b8f01..848c3465fca 100644 --- a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Completion/MarkupTransitionCompletionItemProvider.cs +++ b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Completion/MarkupTransitionCompletionItemProvider.cs @@ -36,7 +36,7 @@ public ImmutableArray GetCompletionItems(RazorCompletionCon if (owner is RazorMetaCodeSyntax { SpanStart: var spanStart, MetaCode: [var metaCodeToken, ..] } && spanStart == context.AbsoluteIndex) { var previousToken = metaCodeToken.GetPreviousToken(); - owner = previousToken.Parent; + owner = previousToken!.Parent; } if (!AtMarkupTransitionCompletionPoint(owner)) diff --git a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Extensions/RazorSyntaxNodeExtensions.cs b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Extensions/RazorSyntaxNodeExtensions.cs index 96a04ec822a..c9a26a09dfc 100644 --- a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Extensions/RazorSyntaxNodeExtensions.cs +++ b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Extensions/RazorSyntaxNodeExtensions.cs @@ -232,7 +232,7 @@ public static LinePositionSpan GetLinePositionSpan(this SyntaxNode node, RazorSo token = token.GetPreviousToken(includeWhitespace); } - var foundPosition = token.Position; + var foundPosition = token!.Position; if (walkMarkersBack && token.Kind == SyntaxKind.Marker) { diff --git a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Formatting/Passes/New/CSharpFormattingPass.CSharpDocumentGenerator.cs b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Formatting/Passes/New/CSharpFormattingPass.CSharpDocumentGenerator.cs index b701b16329e..1f8578d369d 100644 --- a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Formatting/Passes/New/CSharpFormattingPass.CSharpDocumentGenerator.cs +++ b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Formatting/Passes/New/CSharpFormattingPass.CSharpDocumentGenerator.cs @@ -228,7 +228,7 @@ public void Generate() _builder.AppendLine(additionalLinesBuilder.ToString()); } - public override LineInfo Visit(RazorSyntaxNode node) + public override LineInfo Visit(RazorSyntaxNode? node) { // Sometimes we are in a block where we want to do no formatting at all if (_ignoreUntilLine is not null) diff --git a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/SemanticTokens/SemanticTokensVisitor.cs b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/SemanticTokens/SemanticTokensVisitor.cs index f26b3029ee7..4893c1ac88e 100644 --- a/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/SemanticTokens/SemanticTokensVisitor.cs +++ b/src/razor/src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/SemanticTokens/SemanticTokensVisitor.cs @@ -16,16 +16,17 @@ internal sealed class SemanticTokensVisitor : SyntaxWalker { private readonly ImmutableArray.Builder _semanticRanges; private readonly RazorCodeDocument _razorCodeDocument; + private readonly TextSpan _range; private readonly ISemanticTokensLegendService _semanticTokensLegend; private readonly bool _colorCodeBackground; private bool _addRazorCodeModifier; - private SemanticTokensVisitor(ImmutableArray.Builder semanticRanges, RazorCodeDocument razorCodeDocument, TextSpan? range, ISemanticTokensLegendService semanticTokensLegend, bool colorCodeBackground) - : base(range) + private SemanticTokensVisitor(ImmutableArray.Builder semanticRanges, RazorCodeDocument razorCodeDocument, TextSpan range, ISemanticTokensLegendService semanticTokensLegend, bool colorCodeBackground) { _semanticRanges = semanticRanges; _razorCodeDocument = razorCodeDocument; + _range = range; _semanticTokensLegend = semanticTokensLegend; _colorCodeBackground = colorCodeBackground; } @@ -49,6 +50,19 @@ private void Visit(SyntaxList syntaxNodes) } } + private bool IsInRange(TextSpan span) + { + return _range.OverlapsWith(span); + } + + public override void Visit(SyntaxNode? node) + { + if (node != null && IsInRange(node.Span)) + { + base.Visit(node); + } + } + #region HTML public override void VisitMarkupTextLiteral(MarkupTextLiteralSyntax node) @@ -537,6 +551,11 @@ private void AddSemanticRange(SyntaxNode node, int semanticKind) return; } + if (!IsInRange(node.Span)) + { + return; + } + var source = _razorCodeDocument.Source; var range = node.GetLinePositionSpan(source); var tokenModifier = _addRazorCodeModifier ? _semanticTokensLegend.TokenModifiers.RazorCodeModifier : 0; diff --git a/src/razor/src/Razor/test/Microsoft.AspNetCore.Razor.Test.Common.Tooling/Language/Legacy/ToolingParserTestBase.cs b/src/razor/src/Razor/test/Microsoft.AspNetCore.Razor.Test.Common.Tooling/Language/Legacy/ToolingParserTestBase.cs index de1dcae9e6f..2da046af94f 100644 --- a/src/razor/src/Razor/test/Microsoft.AspNetCore.Razor.Test.Common.Tooling/Language/Legacy/ToolingParserTestBase.cs +++ b/src/razor/src/Razor/test/Microsoft.AspNetCore.Razor.Test.Common.Tooling/Language/Legacy/ToolingParserTestBase.cs @@ -73,7 +73,7 @@ internal virtual void AssertSyntaxTreeNodeMatchesBaseline(RazorSyntaxTree syntax { // Write syntax tree baseline var baselineFullPath = Path.Combine(TestProjectRoot, baselineFileName); - File.WriteAllText(baselineFullPath, SyntaxNodeSerializer.Serialize(root, validateSpanEditHandlers: EnableSpanEditHandlers)); + File.WriteAllText(baselineFullPath, TestSyntaxSerializer.Serialize(root, allowSpanEditHandlers: EnableSpanEditHandlers)); // Write diagnostics baseline var baselineDiagnosticsFullPath = Path.Combine(TestProjectRoot, baselineDiagnosticsFileName); @@ -114,7 +114,7 @@ internal virtual void AssertSyntaxTreeNodeMatchesBaseline(RazorSyntaxTree syntax } var syntaxNodeBaseline = stFile.ReadAllText(); - var actualSyntaxNodes = SyntaxNodeSerializer.Serialize(root, validateSpanEditHandlers: EnableSpanEditHandlers); + var actualSyntaxNodes = TestSyntaxSerializer.Serialize(root, allowSpanEditHandlers: EnableSpanEditHandlers); AssertEx.AssertEqualToleratingWhitespaceDifferences(syntaxNodeBaseline, actualSyntaxNodes); // Verify diagnostics diff --git a/src/razor/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/CompletionIntegrationTests.cs b/src/razor/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/CompletionIntegrationTests.cs index 9d92641f528..01c0b7bdc23 100644 --- a/src/razor/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/CompletionIntegrationTests.cs +++ b/src/razor/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/CompletionIntegrationTests.cs @@ -142,6 +142,29 @@ private void IncrementCount() expectedSelectedItemLabel: "style"); } + [IdeFact] + public async Task CompletionCommit_BlazorDirectiveAttribute() + { + await VerifyTypeAndCommitCompletionAsync( + input: """ + @page "/test" + + Test + + + """, + output: """ + @page "/test" + + Test + + + """, + search: "