From 00f2eb02532841111cff22c23114ae34062eabc7 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Sat, 25 Jan 2025 12:47:36 +0100 Subject: [PATCH 01/12] Updated framework target and language version --- samples/PickAll.Sample/PickAll.Sample.csproj | 2 +- src/PickAll/PickAll.csproj | 4 ++-- tests/PickAll.Specs/PickAll.Specs.csproj | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/PickAll.Sample/PickAll.Sample.csproj b/samples/PickAll.Sample/PickAll.Sample.csproj index de7c8ee..387703b 100755 --- a/samples/PickAll.Sample/PickAll.Sample.csproj +++ b/samples/PickAll.Sample/PickAll.Sample.csproj @@ -5,7 +5,7 @@ Exe - net5.0 + net8.0 9.0 gsscoder PickAll diff --git a/src/PickAll/PickAll.csproj b/src/PickAll/PickAll.csproj index 826ea54..f57dd0c 100755 --- a/src/PickAll/PickAll.csproj +++ b/src/PickAll/PickAll.csproj @@ -1,8 +1,8 @@  - netstandard2.0;netcoreapp3.1;net5.0 - 9.0 + netstandard2.1;net8.0 + 13.0 .NET agile and extensible web searching API .NET agile and extensible web searching API 1.3.1 diff --git a/tests/PickAll.Specs/PickAll.Specs.csproj b/tests/PickAll.Specs/PickAll.Specs.csproj index dd508bb..8380608 100755 --- a/tests/PickAll.Specs/PickAll.Specs.csproj +++ b/tests/PickAll.Specs/PickAll.Specs.csproj @@ -4,8 +4,8 @@ - net5.0 - 9.0 + net8.0 + 13.0 false From 74bdca91074a7a69deed485d2c472e5915ed1c02 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Sat, 25 Jan 2025 12:47:47 +0100 Subject: [PATCH 02/12] Upgraded dependencies --- .paket/Paket.Restore.targets | 997 ++++++++++++++++++----------------- paket.dependencies | 32 +- paket.lock | 145 +++-- 3 files changed, 588 insertions(+), 586 deletions(-) diff --git a/.paket/Paket.Restore.targets b/.paket/Paket.Restore.targets index e230bb2..c66062b 100644 --- a/.paket/Paket.Restore.targets +++ b/.paket/Paket.Restore.targets @@ -1,322 +1,325 @@ - - - - - - $(MSBuildAllProjects);$(MSBuildThisFileFullPath) - - $(MSBuildVersion) - 15.0.0 - false - true - - true - $(MSBuildThisFileDirectory) - $(MSBuildThisFileDirectory)..\ - $(PaketRootPath)paket-files\paket.restore.cached - $(PaketRootPath)paket.lock - classic - proj - assembly - native - /Library/Frameworks/Mono.framework/Commands/mono - mono - - - $(PaketRootPath)paket.bootstrapper.exe - $(PaketToolsPath)paket.bootstrapper.exe - $([System.IO.Path]::GetDirectoryName("$(PaketBootStrapperExePath)"))\ - - "$(PaketBootStrapperExePath)" - $(MonoPath) --runtime=v4.0.30319 "$(PaketBootStrapperExePath)" - - - - - true - true - - - True - - - False - - $(BaseIntermediateOutputPath.TrimEnd('\').TrimEnd('\/')) - - - - - - - - - $(PaketRootPath)paket - $(PaketToolsPath)paket - - - - - - $(PaketRootPath)paket.exe - $(PaketToolsPath)paket.exe - - - - - - <_DotnetToolsJson Condition="Exists('$(PaketRootPath)/.config/dotnet-tools.json')">$([System.IO.File]::ReadAllText("$(PaketRootPath)/.config/dotnet-tools.json")) - <_ConfigContainsPaket Condition=" '$(_DotnetToolsJson)' != ''">$(_DotnetToolsJson.Contains('"paket"')) - <_ConfigContainsPaket Condition=" '$(_ConfigContainsPaket)' == ''">false - - - - - - - - - - - <_PaketCommand>dotnet paket - - - - - - $(PaketToolsPath)paket - $(PaketBootStrapperExeDir)paket - - - paket - - - - - <_PaketExeExtension>$([System.IO.Path]::GetExtension("$(PaketExePath)")) - <_PaketCommand Condition=" '$(_PaketCommand)' == '' AND '$(_PaketExeExtension)' == '.dll' ">dotnet "$(PaketExePath)" - <_PaketCommand Condition=" '$(_PaketCommand)' == '' AND '$(OS)' != 'Windows_NT' AND '$(_PaketExeExtension)' == '.exe' ">$(MonoPath) --runtime=v4.0.30319 "$(PaketExePath)" - <_PaketCommand Condition=" '$(_PaketCommand)' == '' ">"$(PaketExePath)" - - - - - - - - - - - - - - - - - - - - - true - $(NoWarn);NU1603;NU1604;NU1605;NU1608 - false - true - - - - - - - - - $([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)')) - - - - - - - $([System.Text.RegularExpressions.Regex]::Split(`%(Identity)`, `": "`)[0].Replace(`"`, ``).Replace(` `, ``)) - $([System.Text.RegularExpressions.Regex]::Split(`%(Identity)`, `": "`)[1].Replace(`"`, ``).Replace(` `, ``)) - - - - - %(PaketRestoreCachedKeyValue.Value) - %(PaketRestoreCachedKeyValue.Value) - - - - - true - false - true - - - + + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + $(MSBuildVersion) + 15.0.0 + false + true + + true + $(MSBuildThisFileDirectory) + $(MSBuildThisFileDirectory)..\ + $(PaketRootPath)paket-files\paket.restore.cached + $(PaketRootPath)paket.lock + classic + proj + assembly + native + /Library/Frameworks/Mono.framework/Commands/mono + mono + + + $(PaketRootPath)paket.bootstrapper.exe + $(PaketToolsPath)paket.bootstrapper.exe + $([System.IO.Path]::GetDirectoryName("$(PaketBootStrapperExePath)"))\ + + "$(PaketBootStrapperExePath)" + $(MonoPath) --runtime=v4.0.30319 "$(PaketBootStrapperExePath)" + + + + + true + true + + + True + + + False + + $(BaseIntermediateOutputPath.TrimEnd('\').TrimEnd('\/')) + + + + + + + + + $(PaketRootPath)paket + $(PaketToolsPath)paket + + + + + + $(PaketRootPath)paket.exe + $(PaketToolsPath)paket.exe + + + + + + <_DotnetToolsJson Condition="Exists('$(PaketRootPath)/.config/dotnet-tools.json')">$([System.IO.File]::ReadAllText("$(PaketRootPath)/.config/dotnet-tools.json")) + <_ConfigContainsPaket Condition=" '$(_DotnetToolsJson)' != ''">$(_DotnetToolsJson.Contains('"paket"')) + <_ConfigContainsPaket Condition=" '$(_ConfigContainsPaket)' == ''">false + + + + + + + + + + + <_PaketCommand>dotnet paket + + + + + + $(PaketToolsPath)paket + $(PaketBootStrapperExeDir)paket + + + paket + + + + + <_PaketExeExtension>$([System.IO.Path]::GetExtension("$(PaketExePath)")) + <_PaketCommand Condition=" '$(_PaketCommand)' == '' AND '$(_PaketExeExtension)' == '.dll' ">dotnet "$(PaketExePath)" + <_PaketCommand Condition=" '$(_PaketCommand)' == '' AND '$(OS)' != 'Windows_NT' AND '$(_PaketExeExtension)' == '.exe' ">$(MonoPath) --runtime=v4.0.30319 "$(PaketExePath)" + <_PaketCommand Condition=" '$(_PaketCommand)' == '' ">"$(PaketExePath)" + + + + + + + + + + + + + + + + + + + + + true + $(NoWarn);NU1603;NU1604;NU1605;NU1608 + false + true + + + + + + + + + $([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)')) + + + + + + + $([System.Text.RegularExpressions.Regex]::Split(`%(Identity)`, `": "`)[0].Replace(`"`, ``).Replace(` `, ``)) + $([System.Text.RegularExpressions.Regex]::Split(`%(Identity)`, `": "`)[1].Replace(`"`, ``).Replace(` `, ``)) + + + + + %(PaketRestoreCachedKeyValue.Value) + %(PaketRestoreCachedKeyValue.Value) + + + + + true + false + true + + + - - true - - - - - - - - - - - - - - - - - - - $(PaketIntermediateOutputPath)\$(MSBuildProjectFile).paket.references.cached - - $(MSBuildProjectFullPath).paket.references - - $(MSBuildProjectDirectory)\$(MSBuildProjectName).paket.references - - $(MSBuildProjectDirectory)\paket.references - - false - true - true - references-file-or-cache-not-found - - - - - $([System.IO.File]::ReadAllText('$(PaketReferencesCachedFilePath)')) - $([System.IO.File]::ReadAllText('$(PaketOriginalReferencesFilePath)')) - references-file - false - - - - - false - - - - - true - target-framework '$(TargetFramework)' or '$(TargetFrameworks)' files @(PaketResolvedFilePaths) - - - - - - - - - - - false - true - - - - - - - - - - - $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',').Length) - $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[0]) - $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[1]) - $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[4]) - $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[5]) - $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[6]) - $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[7]) - - - %(PaketReferencesFileLinesInfo.PackageVersion) - All - runtime - $(ExcludeAssets);contentFiles - $(ExcludeAssets);build;buildMultitargeting;buildTransitive - true - true - - - - - $(PaketIntermediateOutputPath)/$(MSBuildProjectFile).paket.clitools - - - - - - - - - $([System.String]::Copy('%(PaketCliToolFileLines.Identity)').Split(',')[0]) - $([System.String]::Copy('%(PaketCliToolFileLines.Identity)').Split(',')[1]) - - - %(PaketCliToolFileLinesInfo.PackageVersion) - - - - + + + + + + + + + + + + + + + + $(PaketIntermediateOutputPath)\$(MSBuildProjectFile).paket.references.cached + + $(MSBuildProjectFullPath).paket.references + + $(MSBuildProjectDirectory)\$(MSBuildProjectName).paket.references + + $(MSBuildProjectDirectory)\paket.references + + false + true + true + references-file-or-cache-not-found + + + + + $([System.IO.File]::ReadAllText('$(PaketReferencesCachedFilePath)')) + $([System.IO.File]::ReadAllText('$(PaketOriginalReferencesFilePath)')) + references-file + false + + + + + false + + + + + true + target-framework '$(TargetFramework)' or '$(TargetFrameworks)' files @(PaketResolvedFilePaths) + + + + + + + + + + + false + true + + + + + + + + + + + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',').Length) + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[0]) + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[1]) + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[4]) + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[5]) + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[6]) + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[7]) + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[8]) + + + %(PaketReferencesFileLinesInfo.PackageVersion) + All + runtime + $(ExcludeAssets);contentFiles + $(ExcludeAssets);build;buildMultitargeting;buildTransitive + %(PaketReferencesFileLinesInfo.Aliases) + true + true + + + + + + $(PaketIntermediateOutputPath)/$(MSBuildProjectFile).paket.clitools + + + + + + + + + $([System.String]::Copy('%(PaketCliToolFileLines.Identity)').Split(',')[0]) + $([System.String]::Copy('%(PaketCliToolFileLines.Identity)').Split(',')[1]) + + + %(PaketCliToolFileLinesInfo.PackageVersion) + + + + - - - - - false - - - - - - <_NuspecFilesNewLocation Include="$(PaketIntermediateOutputPath)\$(Configuration)\*.nuspec"/> - - - - - - $(MSBuildProjectDirectory)/$(MSBuildProjectFile) - true - false - true - false - true - false - true - false - true - false - true - $(PaketIntermediateOutputPath)\$(Configuration) - $(PaketIntermediateOutputPath) - - - - <_NuspecFiles Include="$(AdjustedNuspecOutputPath)\*.$(PackageVersion.Split(`+`)[0]).nuspec"/> - - - - - - - - - + + + + + false + + + + + + <_NuspecFilesNewLocation Include="$(PaketIntermediateOutputPath)\$(Configuration)\*.nuspec"/> + + + + + + $(MSBuildProjectDirectory)/$(MSBuildProjectFile) + true + false + true + false + true + false + true + false + true + false + true + $(PaketIntermediateOutputPath)\$(Configuration) + $(PaketIntermediateOutputPath) + + + + <_NuspecFiles Include="$(AdjustedNuspecOutputPath)\*.$(PackageVersion.Split(`+`)[0]).nuspec"/> + + + + + + + + + - - - - - - - - - + + + + + + + + + diff --git a/paket.dependencies b/paket.dependencies index 94d4385..e4470e6 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -1,24 +1,24 @@ group main source https://www.nuget.org/api/v2 - framework: netstandard20, netcoreapp31, net50 - nuget AngleSharp 0.14.0 - nuget AngleSharp.Io 0.14.0 - nuget SharpX 1.1.5 + framework: netstandard21, net80 + nuget AngleSharp 1.2.0 + nuget AngleSharp.Io 1.0.0 + nuget SharpX 6.4.6 group specs source https://www.nuget.org/api/v2 - framework: net50 - nuget Microsoft.NET.Test.Sdk 16.9.4 - nuget coverlet.collector 1.0.1 - nuget xunit 2.4.1 - nuget xunit.runner.visualstudio 2.4.3 - nuget FluentAssertions 6.2.0 - nuget SharpX 1.1.5 - nuget Bogus 33.1.1 - nuget WaffleGenerator 4.2.1 - nuget WaffleGenerator.Bogus 4.2.1 + framework: net80 + nuget Microsoft.NET.Test.Sdk 17.12.0 + nuget coverlet.collector 6.0.4 + nuget xunit 2.9.3 + nuget xunit.runner.visualstudio 3.0.1 + nuget FluentAssertions 8.0.1 + nuget SharpX 6.4.6 + nuget Bogus 35.6.1 + nuget WaffleGenerator 4.2.2 + nuget WaffleGenerator.Bogus 4.2.2 group sample source https://www.nuget.org/api/v2 - framework: net50 - nuget CommandLineParser 2.7.82 + framework: net80 + nuget CommandLineParser 2.9.1 diff --git a/paket.lock b/paket.lock index 4675018..7e06a2a 100644 --- a/paket.lock +++ b/paket.lock @@ -1,88 +1,87 @@ -RESTRICTION: || (== net5.0) (== netcoreapp3.1) (== netstandard2.0) +RESTRICTION: || (== net8.0) (== netstandard2.1) NUGET remote: https://www.nuget.org/api/v2 - AngleSharp (0.14) - System.Text.Encoding.CodePages (>= 4.5) - AngleSharp.Io (0.14) - AngleSharp (>= 0.14) - FSharp.Core (6.0.1) - Microsoft.NETCore.Platforms (3.1) - restriction: || (== net5.0) (== netcoreapp3.1) (&& (== netstandard2.0) (>= netcoreapp2.0)) (&& (== netstandard2.0) (>= netcoreapp3.1)) - SharpX (1.1.5) - FSharp.Core (>= 4.7) - System.Runtime.CompilerServices.Unsafe (4.7) - restriction: || (&& (== net5.0) (>= net461)) (&& (== net5.0) (< netcoreapp2.0)) (&& (== net5.0) (< netcoreapp3.1)) (&& (== netcoreapp3.1) (>= net461)) (&& (== netcoreapp3.1) (< netcoreapp2.0)) (== netstandard2.0) - System.Text.Encoding.CodePages (4.7) - Microsoft.NETCore.Platforms (>= 3.1) - restriction: || (== net5.0) (== netcoreapp3.1) (&& (== netstandard2.0) (>= netcoreapp2.0)) (&& (== netstandard2.0) (>= netcoreapp3.1)) - System.Runtime.CompilerServices.Unsafe (>= 4.7) - restriction: || (&& (== net5.0) (>= net461)) (&& (== net5.0) (< netcoreapp2.0)) (&& (== net5.0) (< netcoreapp3.1)) (&& (== netcoreapp3.1) (>= net461)) (&& (== netcoreapp3.1) (< netcoreapp2.0)) (== netstandard2.0) + AngleSharp (1.2) + System.Text.Encoding.CodePages (>= 8.0) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (== netstandard2.1) + AngleSharp.Io (1.0) + AngleSharp (>= 1.0 < 2.0) + FsCheck (3.0.0-rc1) + FSharp.Core (>= 5.0.2) + FSharp.Core (5.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (9.0.1) + Microsoft.Extensions.Logging.Abstractions (8.0.1) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0.1) + System.Buffers (>= 4.5.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (== netstandard2.1) + System.Memory (>= 4.5.5) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (== netstandard2.1) + Microsoft.NETCore.Platforms (7.0.4) + SharpX (6.4.6) + FsCheck (3.0.0-rc1) + FSharp.Core (5.0.2) + Microsoft.Extensions.Logging.Abstractions (8.0.1) + Microsoft.NETCore.Platforms (7.0.4) + System.Buffers (4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) + System.Memory (4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (== netstandard2.1) + System.Buffers (>= 4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) + System.Numerics.Vectors (>= 4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (>= 6.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) + System.Numerics.Vectors (4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (6.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) + System.Text.Encoding.CodePages (9.0.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (== netstandard2.1) + System.Memory (>= 4.5.5) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (>= 6.0) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) GROUP sample -RESTRICTION: == net5.0 +RESTRICTION: == net8.0 NUGET remote: https://www.nuget.org/api/v2 - CommandLineParser (2.7.82) + CommandLineParser (2.9.1) GROUP specs -RESTRICTION: == net5.0 +RESTRICTION: == net8.0 NUGET remote: https://www.nuget.org/api/v2 - Bogus (33.1.1) - coverlet.collector (1.0.1) - FluentAssertions (6.2) - System.Configuration.ConfigurationManager (>= 4.4) - FSharp.Core (6.0.1) - Microsoft.CodeCoverage (17.0) - Microsoft.NET.Test.Sdk (16.9.4) - Microsoft.CodeCoverage (>= 16.9.4) - Microsoft.TestPlatform.TestHost (>= 16.9.4) - Microsoft.NETCore.Platforms (5.0.4) - Microsoft.TestPlatform.ObjectModel (17.0) - NuGet.Frameworks (>= 5.0) + Bogus (35.6.1) + coverlet.collector (6.0.4) + FluentAssertions (8.0.1) + FsCheck (3.0.0-rc1) + FSharp.Core (>= 5.0.2) + FSharp.Core (5.0.2) + Microsoft.CodeCoverage (17.12) + Microsoft.Extensions.DependencyInjection.Abstractions (9.0.1) + Microsoft.Extensions.Logging.Abstractions (8.0.1) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0.1) + Microsoft.NET.Test.Sdk (17.12) + Microsoft.CodeCoverage (>= 17.12) + Microsoft.TestPlatform.TestHost (>= 17.12) + Microsoft.NETCore.Platforms (7.0.4) + Microsoft.TestPlatform.ObjectModel (17.12) System.Reflection.Metadata (>= 1.6) - Microsoft.TestPlatform.TestHost (17.0) - Microsoft.TestPlatform.ObjectModel (>= 17.0) - Newtonsoft.Json (>= 9.0.1) - Microsoft.Win32.SystemEvents (5.0) - Microsoft.NETCore.Platforms (>= 5.0) - NETStandard.Library (2.0.3) - Microsoft.NETCore.Platforms (>= 1.1) + Microsoft.TestPlatform.TestHost (17.12) + Microsoft.TestPlatform.ObjectModel (>= 17.12) + Newtonsoft.Json (>= 13.0.1) Newtonsoft.Json (13.0.1) - NuGet.Frameworks (5.11) - SharpX (1.1.5) - FSharp.Core (>= 4.7) - System.Configuration.ConfigurationManager (5.0) - System.Security.Cryptography.ProtectedData (>= 5.0) - System.Security.Permissions (>= 5.0) - System.Drawing.Common (5.0.2) - Microsoft.Win32.SystemEvents (>= 5.0) + SharpX (6.4.6) + FsCheck (3.0.0-rc1) + FSharp.Core (5.0.2) + Microsoft.Extensions.Logging.Abstractions (8.0.1) + Microsoft.NETCore.Platforms (7.0.4) System.Reflection.Metadata (5.0) - System.Security.AccessControl (5.0) - Microsoft.NETCore.Platforms (>= 5.0) - System.Security.Principal.Windows (>= 5.0) - System.Security.Cryptography.ProtectedData (5.0) - System.Security.Permissions (5.0) - System.Security.AccessControl (>= 5.0) - System.Windows.Extensions (>= 5.0) - System.Security.Principal.Windows (5.0) - System.Windows.Extensions (5.0) - System.Drawing.Common (>= 5.0) - WaffleGenerator (4.2.1) - WaffleGenerator.Bogus (4.2.1) - Bogus (>= 33.0.2) - WaffleGenerator (>= 4.2.1) - xunit (2.4.1) - xunit.analyzers (>= 0.10) - xunit.assert (2.4.1) - xunit.core (2.4.1) + WaffleGenerator (4.2.2) + WaffleGenerator.Bogus (4.2.2) + Bogus (>= 35.6.1) + WaffleGenerator (>= 4.2.2) + xunit (2.9.3) + xunit.analyzers (>= 1.18) + xunit.assert (>= 2.9.3) + xunit.core (2.9.3) xunit.abstractions (2.0.3) - xunit.analyzers (0.10) - xunit.assert (2.4.1) - NETStandard.Library (>= 1.6.1) - xunit.core (2.4.1) - xunit.extensibility.core (2.4.1) - xunit.extensibility.execution (2.4.1) - xunit.extensibility.core (2.4.1) - NETStandard.Library (>= 1.6.1) + xunit.analyzers (1.19) + xunit.assert (2.9.3) + xunit.core (2.9.3) + xunit.extensibility.core (2.9.3) + xunit.extensibility.execution (2.9.3) + xunit.extensibility.core (2.9.3) xunit.abstractions (>= 2.0.3) - xunit.extensibility.execution (2.4.1) - NETStandard.Library (>= 1.6.1) - xunit.extensibility.core (2.4.1) - xunit.runner.visualstudio (2.4.3) + xunit.extensibility.execution (2.9.3) + xunit.extensibility.core (2.9.3) + xunit.runner.visualstudio (3.0.1) From e1c8fbd77e78744323cb04f4f15f225473a4f827 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Sat, 25 Jan 2025 13:53:17 +0100 Subject: [PATCH 03/12] Enabled nullable and fixed some warning --- src/PickAll/Abstractions/Searcher.cs | 2 +- src/PickAll/Abstractions/Service.cs | 8 ++++---- src/PickAll/Internal/ObjectExtensions.cs | 4 ++-- src/PickAll/PickAll.csproj | 1 + src/PickAll/ResultInfoExtensions.cs | 2 +- src/PickAll/SearchContext.cs | 6 +++--- src/PickAll/SearchContextExtensions.cs | 4 ++-- 7 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/PickAll/Abstractions/Searcher.cs b/src/PickAll/Abstractions/Searcher.cs index c2a3330..151ca9f 100644 --- a/src/PickAll/Abstractions/Searcher.cs +++ b/src/PickAll/Abstractions/Searcher.cs @@ -22,7 +22,7 @@ public Searcher(object settings) public abstract Task> SearchAsync(string query); protected ResultInfo CreateResult( - int index, string url, string description, object data = null) + int index, string url, string description, object? data = null) { var result = new ResultInfo(Name, index, url, description, data); EventHelper.RaiseEvent(this, ResultCreated, diff --git a/src/PickAll/Abstractions/Service.cs b/src/PickAll/Abstractions/Service.cs index e47ca91..4350687 100644 --- a/src/PickAll/Abstractions/Service.cs +++ b/src/PickAll/Abstractions/Service.cs @@ -5,10 +5,10 @@ namespace PickAll /// Represents a service managed by SearchContext. public abstract class Service { - SearchContext _context; - internal event EventHandler Load; + SearchContext? _context; + internal event EventHandler? Load; - public SearchContext Context + public SearchContext? Context { get { return _context; } set @@ -17,7 +17,7 @@ public SearchContext Context // Guard against raising load event before configuration happens. // A service is loaded when is bound to a search context. if (_context == null) return; - EventHelper.RaiseEvent(this, Load, EventArgs.Empty, _context.Settings.EnableRaisingEvents); + EventHelper.RaiseEvent(this, Load!, EventArgs.Empty, _context.Settings.EnableRaisingEvents); } } diff --git a/src/PickAll/Internal/ObjectExtensions.cs b/src/PickAll/Internal/ObjectExtensions.cs index adb1a36..be95bf8 100644 --- a/src/PickAll/Internal/ObjectExtensions.cs +++ b/src/PickAll/Internal/ObjectExtensions.cs @@ -12,13 +12,13 @@ public static IEnumerable Add(this IEnumerable collection, T newElement } public static IEnumerable Map(this IEnumerable collection, Func func, - Func predicate = null) + Func? predicate = null) { foreach (var element in collection) { if (element.GetType().EqualsOrSubtype() && (predicate == null || (predicate != null && predicate((T)element)))) { - yield return func((T)element); + yield return func((T)element)!; } else { yield return element; diff --git a/src/PickAll/PickAll.csproj b/src/PickAll/PickAll.csproj index f57dd0c..9d0cfaa 100755 --- a/src/PickAll/PickAll.csproj +++ b/src/PickAll/PickAll.csproj @@ -3,6 +3,7 @@ netstandard2.1;net8.0 13.0 + enable .NET agile and extensible web searching API .NET agile and extensible web searching API 1.3.1 diff --git a/src/PickAll/ResultInfoExtensions.cs b/src/PickAll/ResultInfoExtensions.cs index 9ded4eb..23e4a88 100644 --- a/src/PickAll/ResultInfoExtensions.cs +++ b/src/PickAll/ResultInfoExtensions.cs @@ -2,7 +2,7 @@ namespace PickAll { public static class ResultInfoExtensions { - public static ResultInfo Clone(this ResultInfo resultInfo, object data = null) + public static ResultInfo Clone(this ResultInfo resultInfo, object? data = null) { Guard.AgainstNull(nameof(resultInfo), resultInfo); diff --git a/src/PickAll/SearchContext.cs b/src/PickAll/SearchContext.cs index 40eed51..bba1ec1 100755 --- a/src/PickAll/SearchContext.cs +++ b/src/PickAll/SearchContext.cs @@ -53,7 +53,7 @@ public SearchContext(params Type[] services) : this() Services = Enumerable.Empty(); foreach (var type in services) { - var instance = Activator.CreateInstance(type, new object[] { null }); + var instance = Activator.CreateInstance(type, [null]); Services = Services.Add(instance); } } @@ -164,14 +164,14 @@ static IEnumerable Configure(string query, SearchContext context) return services; } - static HttpClient BuildHttpClient(TimeSpan? timeout, HttpClient defaultClient = null) + static HttpClient BuildHttpClient(TimeSpan? timeout, HttpClient? defaultClient = null) { if (timeout.HasValue) { var client = new HttpClient(); client.Timeout = timeout.Value; return client; } - return defaultClient; + return defaultClient!; } static IBrowsingContext BuildBrowsingContext( diff --git a/src/PickAll/SearchContextExtensions.cs b/src/PickAll/SearchContextExtensions.cs index a4efa44..5516305 100644 --- a/src/PickAll/SearchContextExtensions.cs +++ b/src/PickAll/SearchContextExtensions.cs @@ -9,7 +9,7 @@ public static class SearchContextExtensions { /// Registers an instance of Searcher or PostProcessor without settings, /// using its type. - public static SearchContext With(this SearchContext context, object settings = null) + public static SearchContext With(this SearchContext context, object? settings = null) where T : Service { Guard.AgainstNull(nameof(context), context); @@ -23,7 +23,7 @@ public static SearchContext With(this SearchContext context, object settings /// Registers an instance of Searcher or PostProcessor without settings, /// using its type name. public static SearchContext With(this SearchContext context, string serviceName, - object settings = null) + object? settings = null) { Guard.AgainstNull(nameof(context), context); Guard.AgainstNull(nameof(serviceName), serviceName); From ef47496f381205253f27f8d49d5a57683ae35c23 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Sat, 25 Jan 2025 18:21:10 +0100 Subject: [PATCH 04/12] Added Puppeteer browser context to SearchContext --- paket.dependencies | 1 + paket.lock | 43 ++++++++++++++++++++++++++++++- src/PickAll/SearchContext.cs | 49 +++++++++++++++++++++++++++++++++--- src/PickAll/paket.references | 1 + 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/paket.dependencies b/paket.dependencies index e4470e6..ae84bd4 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -4,6 +4,7 @@ group main nuget AngleSharp 1.2.0 nuget AngleSharp.Io 1.0.0 nuget SharpX 6.4.6 + nuget PuppeteerSharp 20.0.5 group specs source https://www.nuget.org/api/v2 diff --git a/paket.lock b/paket.lock index 7e06a2a..9e11fc2 100644 --- a/paket.lock +++ b/paket.lock @@ -8,27 +8,68 @@ NUGET FsCheck (3.0.0-rc1) FSharp.Core (>= 5.0.2) FSharp.Core (5.0.2) + Microsoft.Bcl.AsyncInterfaces (9.0.1) - restriction: == netstandard2.1 + Microsoft.Extensions.DependencyInjection (9.0.1) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0.1) Microsoft.Extensions.DependencyInjection.Abstractions (9.0.1) + Microsoft.Extensions.Logging (8.0) + Microsoft.Extensions.DependencyInjection (>= 8.0) + Microsoft.Extensions.Logging.Abstractions (>= 8.0) + Microsoft.Extensions.Options (>= 8.0) + System.Diagnostics.DiagnosticSource (>= 8.0) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (&& (== net8.0) (< netstandard2.1)) (== netstandard2.1) Microsoft.Extensions.Logging.Abstractions (8.0.1) Microsoft.Extensions.DependencyInjection.Abstractions (>= 8.0.1) System.Buffers (>= 4.5.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (== netstandard2.1) System.Memory (>= 4.5.5) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (== netstandard2.1) + Microsoft.Extensions.Options (9.0.1) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0.1) + Microsoft.Extensions.Primitives (>= 9.0.1) + System.ComponentModel.Annotations (>= 5.0) - restriction: || (&& (== net8.0) (< netstandard2.1)) (== netstandard2.1) + Microsoft.Extensions.Primitives (9.0.1) + System.Memory (>= 4.5.5) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (>= 6.0) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) Microsoft.NETCore.Platforms (7.0.4) + PuppeteerSharp (20.0.5) + Microsoft.Bcl.AsyncInterfaces (>= 8.0) - restriction: == netstandard2.1 + Microsoft.Extensions.Logging (>= 8.0) + System.Text.Json (>= 8.0.5) - restriction: == netstandard2.1 SharpX (6.4.6) FsCheck (3.0.0-rc1) FSharp.Core (5.0.2) Microsoft.Extensions.Logging.Abstractions (8.0.1) Microsoft.NETCore.Platforms (7.0.4) System.Buffers (4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) + System.ComponentModel.Annotations (5.0) - restriction: || (&& (== net8.0) (< netstandard2.1)) (== netstandard2.1) + System.Diagnostics.DiagnosticSource (9.0.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (&& (== net8.0) (< netstandard2.1)) (== netstandard2.1) + System.Memory (>= 4.5.5) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (>= 6.0) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.IO.Pipelines (9.0.1) - restriction: == netstandard2.1 + System.Buffers (>= 4.5.1) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Memory (>= 4.5.5) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Threading.Tasks.Extensions (>= 4.5.4) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) System.Memory (4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (== netstandard2.1) System.Buffers (>= 4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) System.Numerics.Vectors (>= 4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) System.Numerics.Vectors (4.6) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) - System.Runtime.CompilerServices.Unsafe (6.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (6.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) System.Text.Encoding.CodePages (9.0.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< net6.0)) (== netstandard2.1) System.Memory (>= 4.5.5) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Text.Encodings.Web (9.0.1) - restriction: == netstandard2.1 + System.Buffers (>= 4.5.1) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Memory (>= 4.5.5) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (>= 6.0) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Text.Json (9.0.1) - restriction: == netstandard2.1 + Microsoft.Bcl.AsyncInterfaces (>= 9.0.1) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Buffers (>= 4.5.1) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.IO.Pipelines (>= 9.0.1) + System.Memory (>= 4.5.5) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Runtime.CompilerServices.Unsafe (>= 6.0) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Text.Encodings.Web (>= 9.0.1) + System.Threading.Tasks.Extensions (>= 4.5.4) - restriction: || (&& (== net8.0) (>= net462)) (== netstandard2.1) + System.Threading.Tasks.Extensions (4.6) - restriction: == netstandard2.1 + System.Runtime.CompilerServices.Unsafe (>= 6.1) - restriction: || (&& (== net8.0) (>= net462)) (&& (== net8.0) (< netcoreapp2.1)) (== netstandard2.1) GROUP sample RESTRICTION: == net8.0 diff --git a/src/PickAll/SearchContext.cs b/src/PickAll/SearchContext.cs index bba1ec1..e8b6085 100755 --- a/src/PickAll/SearchContext.cs +++ b/src/PickAll/SearchContext.cs @@ -5,16 +5,19 @@ using System.Net.Http; using AngleSharp; using AngleSharp.Io.Network; +using PuppeteerSharp; namespace PickAll { /// Manages Searcher and PostProcessor instances to gather /// and elaborate results. - public sealed class SearchContext - { + public sealed class SearchContext : IDisposable + { + bool _disposed = false; readonly Lazy _browsing; readonly Lazy _fetching; - static readonly Lazy _default = new Lazy( + readonly Lazy _headlessBrowsing; + static readonly Lazy _default = new( () => new SearchContext( typeof(Google), typeof(DuckDuckGo), @@ -29,6 +32,8 @@ internal SearchContext(IEnumerable services, ContextSettings settings) () => BuildBrowsingContext(settings.Timeout, () => BuildHttpClient(settings.Timeout))); _fetching = new Lazy( () => new FetchingContext(BuildHttpClient(settings.Timeout, new HttpClient()))); + _headlessBrowsing = new Lazy( + () => BuildHeadlessBrowsingContextAsync().Result); #if DEBUG EnforceMaximumResults = true; #endif @@ -74,10 +79,12 @@ public SearchContext(params Type[] services) : this() #pragma warning restore CS3003 /// Current IFetchingContext instance. public IFetchingContext Fetching => _fetching.Value; + /// Current IPage instance. + public IPage HeadlessPage => _headlessPage.Value; #if !DEBUG internal IEnumerable Services { get; private set; } internal ContextSettings Settings { get; private set; } - #else +#else public IEnumerable Services { get; private set; } public ContextSettings Settings { get; private set; } public bool EnforceMaximumResults { get; set; } // Debug only @@ -186,5 +193,39 @@ static IBrowsingContext BuildBrowsingContext( } return BrowsingContext.New(Configuration.Default.WithDefaultLoader()); } + + static async Task BuildHeadlessBrowsingContextAsync() + { + var browserFetcher = new BrowserFetcher(); + await browserFetcher.DownloadAsync(); + var browser = await Puppeteer.LaunchAsync( + new LaunchOptions { Headless = true }); + return browser; + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + void Dispose(bool disposing) + { + if (_disposed) + return; + + if (disposing) { + if (_headlessBrowsing.IsValueCreated) { + _headlessBrowsing.Value.Dispose(); + } + } + + _disposed = true; + } + + ~SearchContext() + { + Dispose(false); + } } } diff --git a/src/PickAll/paket.references b/src/PickAll/paket.references index 73058dc..4764bcd 100644 --- a/src/PickAll/paket.references +++ b/src/PickAll/paket.references @@ -2,3 +2,4 @@ group main AngleSharp AngleSharp.Io SharpX + PuppeteerSharp From ab5cd805ab52737159bcefc9be34931ca6f50a65 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Sun, 26 Jan 2025 05:14:01 +0100 Subject: [PATCH 05/12] Enabled explicit usings in main project --- src/PickAll/PickAll.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PickAll/PickAll.csproj b/src/PickAll/PickAll.csproj index 9d0cfaa..72d3dc4 100755 --- a/src/PickAll/PickAll.csproj +++ b/src/PickAll/PickAll.csproj @@ -4,6 +4,7 @@ netstandard2.1;net8.0 13.0 enable + enable .NET agile and extensible web searching API .NET agile and extensible web searching API 1.3.1 From d34376ec7535e44fc9107d1c48388a212b693980 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Sun, 26 Jan 2025 06:35:30 +0100 Subject: [PATCH 06/12] Fixed ambiguous call --- src/PickAll/PostProcessors/Uniqueness.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/PickAll/PostProcessors/Uniqueness.cs b/src/PickAll/PostProcessors/Uniqueness.cs index bbdeed5..6834376 100644 --- a/src/PickAll/PostProcessors/Uniqueness.cs +++ b/src/PickAll/PostProcessors/Uniqueness.cs @@ -1,4 +1,3 @@ -using System.Collections.Generic; using SharpX.Extensions; namespace PickAll @@ -12,7 +11,7 @@ public Uniqueness(object settings) : base(settings) public override IEnumerable Process(IEnumerable results) { - return results.DistinctBy(result => result.Url); + return EnumerableExtensions.DistinctBy(results, result => result.Url); } } } From 536345309d9d5278babf19bcff893369ace77c85 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Wed, 29 Jan 2025 19:02:04 +0100 Subject: [PATCH 07/12] Added HtmlAgilityPack to main project dependencies --- paket.dependencies | 1 + paket.lock | 1 + src/PickAll/paket.references | 1 + 3 files changed, 3 insertions(+) diff --git a/paket.dependencies b/paket.dependencies index ae84bd4..3314dc0 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -5,6 +5,7 @@ group main nuget AngleSharp.Io 1.0.0 nuget SharpX 6.4.6 nuget PuppeteerSharp 20.0.5 + nuget HtmlAgilityPack 1.11.72 group specs source https://www.nuget.org/api/v2 diff --git a/paket.lock b/paket.lock index 9e11fc2..7587e95 100644 --- a/paket.lock +++ b/paket.lock @@ -8,6 +8,7 @@ NUGET FsCheck (3.0.0-rc1) FSharp.Core (>= 5.0.2) FSharp.Core (5.0.2) + HtmlAgilityPack (1.11.72) Microsoft.Bcl.AsyncInterfaces (9.0.1) - restriction: == netstandard2.1 Microsoft.Extensions.DependencyInjection (9.0.1) Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0.1) diff --git a/src/PickAll/paket.references b/src/PickAll/paket.references index 4764bcd..978c39c 100644 --- a/src/PickAll/paket.references +++ b/src/PickAll/paket.references @@ -3,3 +3,4 @@ group main AngleSharp.Io SharpX PuppeteerSharp + HtmlAgilityPack From 3c4dde225e717d1229bc2412d12e769a7685af64 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Wed, 29 Jan 2025 19:03:14 +0100 Subject: [PATCH 08/12] Defined SearcherException class --- src/PickAll/Searchers/SearcherException.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/PickAll/Searchers/SearcherException.cs diff --git a/src/PickAll/Searchers/SearcherException.cs b/src/PickAll/Searchers/SearcherException.cs new file mode 100644 index 0000000..00bc0d9 --- /dev/null +++ b/src/PickAll/Searchers/SearcherException.cs @@ -0,0 +1,12 @@ +namespace PickAll.Searchers; + +public class SearcherException : Exception +{ + public SearcherException(string message): base(message) + { + } + + public SearcherException(string message,Exception innerException) : base(message, innerException) + { + } +} From 2397b02d858f11cb6f61c7a6e3ee92aae1e2e481 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Wed, 29 Jan 2025 19:05:14 +0100 Subject: [PATCH 09/12] Implemented EnsureSuccessOrThrow extension method --- src/PickAll/Internal/ResponseExtensions.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/PickAll/Internal/ResponseExtensions.cs diff --git a/src/PickAll/Internal/ResponseExtensions.cs b/src/PickAll/Internal/ResponseExtensions.cs new file mode 100644 index 0000000..9b9a241 --- /dev/null +++ b/src/PickAll/Internal/ResponseExtensions.cs @@ -0,0 +1,13 @@ +using System.Net; +using PuppeteerSharp; + +static class ResponseExtensions +{ + public static void EnsureSuccessOrThrow(this IResponse response, T exception) where T : Exception + { + var isSuccessStatus = response.Status >= HttpStatusCode.OK && + response.Status <= (HttpStatusCode)299; + + if (!isSuccessStatus) throw exception; + } +} From bb128740167be3b3f5f06397147464702ba2c311 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Wed, 29 Jan 2025 19:05:57 +0100 Subject: [PATCH 10/12] Implemented extension to set Puppeteer in stealth mode --- src/PickAll/Internal/PageExtensions.cs | 102 +++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 src/PickAll/Internal/PageExtensions.cs diff --git a/src/PickAll/Internal/PageExtensions.cs b/src/PickAll/Internal/PageExtensions.cs new file mode 100644 index 0000000..dcc8181 --- /dev/null +++ b/src/PickAll/Internal/PageExtensions.cs @@ -0,0 +1,102 @@ +using PuppeteerSharp; +using SharpX.Extensions; + +static class PageExtensions +{ + static string[] _desktopUserAgents = new[] + { + // Chrome User Agents + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.138 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.138 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.5563.146 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5414.120 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.133 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.133 Safari/537.36", + + // Firefox User Agents + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:114.0) Gecko/20100101 Firefox/114.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11.0; rv:110.0) Gecko/20100101 Firefox/110.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:116.0) Gecko/20100101 Firefox/116.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:116.0) Gecko/20100101 Firefox/116.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/109.0", + "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:113.0) Gecko/20100101 Firefox/113.0", + + // Edge User Agents + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.138 Safari/537.36 Edg/112.0.1722.48", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.77 Safari/537.36 Edg/110.0.1587.69", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.133 Safari/537.36 Edg/114.0.1823.79", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.133 Safari/537.36 Edg/114.0.1823.79", + + // Safari User Agents + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.2 Safari/605.1.15", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15", + + // Additional Desktop User Agents + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.126 Safari/537.36" + }; + + public static async Task UseStealthMode(this IPage page) + { + await page.SetRequestInterceptionAsync(true); + page.Request += async (sender, e) => + { + if (e.Request.IsNavigationRequest) { + var randomUserAgent = _desktopUserAgents.Shuffle().Choice(); + + var headers = new Dictionary(e.Request.Headers) + { + ["User-Agent"] = randomUserAgent + }; + + await e.Request.ContinueAsync(new Payload { + Headers = headers + }); + } + else { + await e.Request.ContinueAsync(); + } + }; + + await page.EvaluateExpressionOnNewDocumentAsync(@" + // Remove navigator.webdriver + Object.defineProperty(navigator, 'webdriver', { get: () => false }); + + // Mock navigator.languages + Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); + + // Mock navigator.plugins + Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] }); + + // Mock window.chrome + Object.defineProperty(window, 'chrome', { get: () => undefined }); + + // Mock permissions query + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => + parameters.name === 'notifications' + ? Promise.resolve({ state: 'denied' }) + : originalQuery(parameters); + + // Mock screen dimensions + Object.defineProperty(window.screen, 'availWidth', { get: () => 1920 }); + Object.defineProperty(window.screen, 'availHeight', { get: () => 1080 }); + + // Add random human-like mouse movements + const simulateMouseMovement = () => { + const event = new MouseEvent('mousemove', { + clientX: Math.random() * window.innerWidth, + clientY: Math.random() * window.innerHeight + }); + document.dispatchEvent(event); + }; + setInterval(simulateMouseMovement, 1000); + "); + + } +} + From 1bf1662b20ad1459c34634400582194e398ead3b Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Wed, 29 Jan 2025 19:06:35 +0100 Subject: [PATCH 11/12] Refactored access to Puppeteer in SearchContext --- src/PickAll/SearchContext.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/PickAll/SearchContext.cs b/src/PickAll/SearchContext.cs index e8b6085..bd92304 100755 --- a/src/PickAll/SearchContext.cs +++ b/src/PickAll/SearchContext.cs @@ -74,13 +74,13 @@ public SearchContext(params Type[] services) : this() /// Occurs a ResultInfo is processed. public event EventHandler ResultProcessed; #pragma warning disable CS3003 - /// Current IBrowsingContext instance. + /// Current AngleSharp IBrowsingContext instance. public IBrowsingContext Browsing => _browsing.Value; #pragma warning restore CS3003 /// Current IFetchingContext instance. public IFetchingContext Fetching => _fetching.Value; - /// Current IPage instance. - public IPage HeadlessPage => _headlessPage.Value; + /// Current PuppeteerSharp IBrowser instance. + public IBrowser HeadlessBrowsing => _headlessBrowsing.Value; #if !DEBUG internal IEnumerable Services { get; private set; } internal ContextSettings Settings { get; private set; } @@ -199,7 +199,9 @@ static IBrowsingContext BuildBrowsingContext( var browserFetcher = new BrowserFetcher(); await browserFetcher.DownloadAsync(); var browser = await Puppeteer.LaunchAsync( - new LaunchOptions { Headless = true }); + new LaunchOptions { + Args = new[] { "--disable-blink-features=AutomationControlled" }, + Headless = true }); return browser; } From d3bf4ef0af7c891480a9ed12ca89a288dd691aa7 Mon Sep 17 00:00:00 2001 From: Giacomo Stelluti Scala Date: Wed, 29 Jan 2025 19:07:57 +0100 Subject: [PATCH 12/12] Re-implemented Bing searcher with Puppeteer Sharp --- samples/PickAll.Sample/PickAll.Sample.csproj | 2 +- src/PickAll/Searchers/Bing.cs | 58 +++++++++++--------- src/PickAll/Searchers/Google.cs | 6 +- tests/PickAll.Specs/Fakes/TextPicker.cs | 41 ++++++++++++++ tests/PickAll.Specs/PickAll.Specs.csproj | 5 +- tests/PickAll.Specs/Tests/SearchersSpecs.cs | 19 +++++++ 6 files changed, 99 insertions(+), 32 deletions(-) create mode 100644 tests/PickAll.Specs/Fakes/TextPicker.cs create mode 100644 tests/PickAll.Specs/Tests/SearchersSpecs.cs diff --git a/samples/PickAll.Sample/PickAll.Sample.csproj b/samples/PickAll.Sample/PickAll.Sample.csproj index 387703b..28562a5 100755 --- a/samples/PickAll.Sample/PickAll.Sample.csproj +++ b/samples/PickAll.Sample/PickAll.Sample.csproj @@ -6,7 +6,7 @@ Exe net8.0 - 9.0 + 13.0 gsscoder PickAll gsscoder diff --git a/src/PickAll/Searchers/Bing.cs b/src/PickAll/Searchers/Bing.cs index 718cb6e..820d653 100644 --- a/src/PickAll/Searchers/Bing.cs +++ b/src/PickAll/Searchers/Bing.cs @@ -1,35 +1,39 @@ -using System; -using System.Threading.Tasks; -using System.Collections.Generic; -using System.Linq; -using AngleSharp; -using AngleSharp.Html.Dom; -using AngleSharp.Dom; +using System.Data; +using HtmlAgilityPack; +using PickAll.Searchers; +using PuppeteerSharp; +using SharpX.Extensions; -namespace PickAll +namespace PickAll; + +/// Searcher that searches on Bing search engine. +public class Bing(object settings) : Searcher(settings) { - /// Searcher that searches on Bing search engine. - public class Bing : Searcher + public override async Task> SearchAsync(string query) { - public Bing(object settings) : base(settings) + var page = await Context!.HeadlessBrowsing.NewPageAsync(); + await page.UseStealthMode(); + var url = $"https://www.bing.com/search?q={Uri.EscapeDataString(query)}"; + var response = await page.GoToAsync(url, new NavigationOptions { + WaitUntil = [WaitUntilNavigation.Load] + }); + response.EnsureSuccessOrThrow( + new SearcherException("Unable to navigate to 'https://www.bing.com/search?q={query}'.")); + + var olHtmlContent = await page.EvaluateExpressionAsync("document.querySelector('ol#b_results').outerHTML"); + if (olHtmlContent == null) { + throw new SearcherException($"Unable to select item 'b_results'."); } + var resultsHtml = new HtmlDocument(); + resultsHtml.LoadHtml(olHtmlContent); - public override async Task> SearchAsync(string query) - { - using var document = await Context.Browsing.OpenAsync("https://www.bing.com/"); - var form = document.QuerySelector("#sb_form"); - ((IHtmlInputElement)form["sb_form_q"]).Value = query; - using var result = await form.SubmitAsync(form); - // Select only actual results - var links = from link in result.QuerySelectorAll("li.b_algo a") - where link.Attributes["href"].Value.StartsWith( - "http", - StringComparison.OrdinalIgnoreCase) - select link; + var links = resultsHtml.DocumentNode.SelectNodes("//li[@class='b_algo']//a") + .Where(x => !x.InnerText.IsEmpty() && !x.InnerText.EqualsIgnoreCase("div") && + !x.Attributes["href"].Value.ContainsIgnoreCase("javascript:")); - return links.Select((link, index) => - CreateResult((ushort)index, link.Attributes["href"].Value, link.Text)); - } + var results = links.Select((link, index) => + CreateResult((ushort)index, link.Attributes["href"].Value, link.InnerText)); + return results; } -} +} \ No newline at end of file diff --git a/src/PickAll/Searchers/Google.cs b/src/PickAll/Searchers/Google.cs index c7e5d0c..4785199 100644 --- a/src/PickAll/Searchers/Google.cs +++ b/src/PickAll/Searchers/Google.cs @@ -14,7 +14,7 @@ public class Google : Searcher { static readonly Regex _normalize = new Regex(@"^/url\?q=([^&]*)&.*", RegexOptions.Compiled); - public Google(object settings) : base(settings) + public Google(object settings) : base(settings) { } @@ -42,7 +42,7 @@ static bool Validate(string url) => url.StartsWith( "/url?", StringComparison.OrdinalIgnoreCase) && !url.StartsWith( - "/url?q=http://webcache.googleusercontent.com",StringComparison.OrdinalIgnoreCase); + "/url?q=http://webcache.googleusercontent.com", StringComparison.OrdinalIgnoreCase); static string Normalize(string url) { @@ -50,4 +50,4 @@ static string Normalize(string url) return match.Groups.Count == 2 ? match.Groups[1].Value : url; } } -} +} \ No newline at end of file diff --git a/tests/PickAll.Specs/Fakes/TextPicker.cs b/tests/PickAll.Specs/Fakes/TextPicker.cs new file mode 100644 index 0000000..a785667 --- /dev/null +++ b/tests/PickAll.Specs/Fakes/TextPicker.cs @@ -0,0 +1,41 @@ +using SharpX.Extensions; + +static class TextPicker +{ + static readonly string[] _sentences = + [ + "info about", + "facts about", + "insides about", + "life of", + "career of", + "private life of", + "public life of", + "curiosities about", + "fake news regarding", + ]; + + static readonly string[] _famousTechNames = + [ + "Elon Musk", + "Larry Ellison", + "Bill Gates", + "Jeff Bezos", + "Mark Zuckerberg", + "Tim Cook", + "Sundar Pichai", + "Satya Nadella", + "Sheryl Sandberg", + "Susan Wojcicki", + "Jack Dorsey", + "Reed Hastings", + "Steve Wozniak", + "Linus Torvalds", + "Marc Andreessen", + "Peter Thiel", + ]; + + public static string GetName() => _famousTechNames.Shuffle().Choice(); + + public static string GetSentence() => _sentences.Shuffle().Choice(); +} diff --git a/tests/PickAll.Specs/PickAll.Specs.csproj b/tests/PickAll.Specs/PickAll.Specs.csproj index 8380608..14dee02 100755 --- a/tests/PickAll.Specs/PickAll.Specs.csproj +++ b/tests/PickAll.Specs/PickAll.Specs.csproj @@ -6,7 +6,10 @@ net8.0 13.0 + enable + enable false + enable - + \ No newline at end of file diff --git a/tests/PickAll.Specs/Tests/SearchersSpecs.cs b/tests/PickAll.Specs/Tests/SearchersSpecs.cs new file mode 100644 index 0000000..d2a2988 --- /dev/null +++ b/tests/PickAll.Specs/Tests/SearchersSpecs.cs @@ -0,0 +1,19 @@ +using PickAll; +using Xunit; + +namespace Tests; + +public class SearchersTests +{ + [Fact] + public async Task Test_Bing() + { + using var sut = new SearchContext() + .With("Bing"); + + var query = $"{TextPicker.GetSentence()} {TextPicker.GetName()}"; + var results = await sut.SearchAsync($"{TextPicker.GetSentence()} {TextPicker.GetName()}"); + + Assert.NotEmpty(results); + } +}