From cce053deafd9fd7d75f057bf71ed177f308a0014 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 12 Jul 2023 10:21:59 -0700 Subject: [PATCH 01/47] Reset release notes --- .../Worker.Extensions.Abstractions/release_notes.md | 4 ++-- release_notes.md | 12 +++++------- sdk/release_notes.md | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/extensions/Worker.Extensions.Abstractions/release_notes.md b/extensions/Worker.Extensions.Abstractions/release_notes.md index 2be5bbc37..e3ed71348 100644 --- a/extensions/Worker.Extensions.Abstractions/release_notes.md +++ b/extensions/Worker.Extensions.Abstractions/release_notes.md @@ -4,6 +4,6 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Extensions.Abstractions 1.3.0 +### Microsoft.Azure.Functions.Worker.Extensions.Abstractions -- Add SupportsDeferredBinding attribute +- diff --git a/release_notes.md b/release_notes.md index 7263c6129..37057e890 100644 --- a/release_notes.md +++ b/release_notes.md @@ -4,16 +4,14 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker (metapackage) 1.18.0 +### Microsoft.Azure.Functions.Worker (metapackage) - -### Microsoft.Azure.Functions.Worker.Core 1.14.0 +### Microsoft.Azure.Functions.Worker.Core -- Unsealed `InputConverterAttribute`. Implementers can now derive from this type to map attributes to custom converters. (#1712) -- Add support for deferred binding (#1676) -- Add binding attribute to converter context (#1660) +- -### Microsoft.Azure.Functions.Worker.Grpc 1.12.0 +### Microsoft.Azure.Functions.Worker.Grpc -- Add support for deferred binding (#1676) +- diff --git a/sdk/release_notes.md b/sdk/release_notes.md index 3723eadc1..52e2b8857 100644 --- a/sdk/release_notes.md +++ b/sdk/release_notes.md @@ -4,9 +4,9 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Sdk 1.12.0 (meta package) +### Microsoft.Azure.Functions.Worker.Sdk -- Add support for deferred binding (#1676) +- ### Microsoft.Azure.Functions.Worker.Sdk.Analyzers (delete if not updated) From 4e8b57a29039b8f7d4bd37b891fab15df52ec6da Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 22 Feb 2023 16:40:38 -0800 Subject: [PATCH 02/47] Add analyzer for SupportsDeferredBindingAttribute (#1367) --- DotNetWorker.sln | 2 +- sdk/Sdk.Analyzers/Constants.cs | 5 +- .../DeferredBindingAttributeNotSupported.cs | 58 ++++++++ sdk/Sdk.Analyzers/DiagnosticDescriptors.cs | 11 +- .../AttributeDataExtensions.cs | 19 ++- .../MethodSymbolExtensions.cs | 0 .../ParameterSymbolExtensions.cs | 0 .../{ => Extensions}/TypeSymbolExtensions.cs | 0 .../WebJobsAttributesNotSupported.cs | 4 +- sdk/release_notes.md | 4 +- .../AsyncVoidAnalyzerTests.cs | 12 +- ...ferredBindingAttributeNotSupportedTests.cs | 134 ++++++++++++++++++ .../Sdk.Analyzers.Tests.csproj | 6 +- .../WebJobsAttributesNotSupportedTests.cs | 14 +- 14 files changed, 241 insertions(+), 28 deletions(-) create mode 100644 sdk/Sdk.Analyzers/DeferredBindingAttributeNotSupported.cs rename sdk/Sdk.Analyzers/{ => Extensions}/AttributeDataExtensions.cs (64%) rename sdk/Sdk.Analyzers/{ => Extensions}/MethodSymbolExtensions.cs (100%) rename sdk/Sdk.Analyzers/{ => Extensions}/ParameterSymbolExtensions.cs (100%) rename sdk/Sdk.Analyzers/{ => Extensions}/TypeSymbolExtensions.cs (100%) rename test/Sdk.Analyzers.Tests/{Sdk.Analyzers.Tests => }/AsyncVoidAnalyzerTests.cs (96%) create mode 100644 test/Sdk.Analyzers.Tests/DeferredBindingAttributeNotSupportedTests.cs rename test/Sdk.Analyzers.Tests/{Sdk.Analyzers.Tests => }/Sdk.Analyzers.Tests.csproj (78%) rename test/Sdk.Analyzers.Tests/{Sdk.Analyzers.Tests => }/WebJobsAttributesNotSupportedTests.cs (95%) diff --git a/DotNetWorker.sln b/DotNetWorker.sln index 90357deb1..dac4af694 100644 --- a/DotNetWorker.sln +++ b/DotNetWorker.sln @@ -70,7 +70,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.Kafka", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sdk.Analyzers", "sdk\Sdk.Analyzers\Sdk.Analyzers.csproj", "{055D602D-D2B3-416B-AC59-1972D832032A}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sdk.Analyzers.Tests", "test\Sdk.Analyzers.Tests\Sdk.Analyzers.Tests\Sdk.Analyzers.Tests.csproj", "{A75EA1E1-2801-460C-87C0-DE6A82D30851}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sdk.Analyzers.Tests", "test\Sdk.Analyzers.Tests\Sdk.Analyzers.Tests.csproj", "{A75EA1E1-2801-460C-87C0-DE6A82D30851}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{083592CA-7DAB-44CE-8979-44FAFA46AEC3}" EndProject diff --git a/sdk/Sdk.Analyzers/Constants.cs b/sdk/Sdk.Analyzers/Constants.cs index b9ecc9db6..b526a6363 100644 --- a/sdk/Sdk.Analyzers/Constants.cs +++ b/sdk/Sdk.Analyzers/Constants.cs @@ -9,7 +9,10 @@ internal static class Types { public const string WorkerFunctionAttribute = "Microsoft.Azure.Functions.Worker.FunctionAttribute"; public const string WebJobsBindingAttribute = "Microsoft.Azure.WebJobs.Description.BindingAttribute"; - + public const string SupportsDeferredBindingAttribute = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.SupportsDeferredBindingAttribute"; + public const string InputBindingAttribute = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.InputBindingAttribute"; + public const string TriggerBindingAttribute = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.TriggerBindingAttribute"; + // System types internal const string TaskType = "System.Threading.Tasks.Task"; } diff --git a/sdk/Sdk.Analyzers/DeferredBindingAttributeNotSupported.cs b/sdk/Sdk.Analyzers/DeferredBindingAttributeNotSupported.cs new file mode 100644 index 000000000..459e1b2ce --- /dev/null +++ b/sdk/Sdk.Analyzers/DeferredBindingAttributeNotSupported.cs @@ -0,0 +1,58 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers +{ + [DiagnosticAnalyzer(LanguageNames.CSharp)] + public class DeferredBindingAttributeNotSupported : DiagnosticAnalyzer + { + public override ImmutableArray SupportedDiagnostics { get { return ImmutableArray.Create(DiagnosticDescriptors.DeferredBindingAttributeNotSupported); } } + + public override void Initialize(AnalysisContext context) + { + context.EnableConcurrentExecution(); + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.Analyze); + context.RegisterSymbolAction(AnalyzeMethod, SymbolKind.NamedType); + } + + private static void AnalyzeMethod(SymbolAnalysisContext symbolAnalysisContext) + { + var symbol = (INamedTypeSymbol)symbolAnalysisContext.Symbol; + + var attributes = symbol.GetAttributes(); + + if (attributes.IsEmpty) + { + return; + } + + foreach (var attribute in attributes) + { + if (attribute.IsSupportsDeferredBindingAttribute() && !IsInputOrTriggerBinding(symbol)) + { + var location = Location.Create(attribute.ApplicationSyntaxReference.SyntaxTree, attribute.ApplicationSyntaxReference.Span); + var diagnostic = Diagnostic.Create(DiagnosticDescriptors.DeferredBindingAttributeNotSupported, location, attribute.AttributeClass.Name); + symbolAnalysisContext.ReportDiagnostic(diagnostic); + } + } + } + + private static bool IsInputOrTriggerBinding(INamedTypeSymbol symbol) + { + var baseType = symbol.BaseType?.ToDisplayString(); + + if (string.Equals(baseType,Constants.Types.InputBindingAttribute, StringComparison.Ordinal) + || string.Equals(baseType,Constants.Types.TriggerBindingAttribute, StringComparison.Ordinal)) + { + return true; + } + + return false; + } + } +} diff --git a/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs b/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs index 613dfc39c..2a094e367 100644 --- a/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs +++ b/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs @@ -1,9 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; -using System.Collections.Generic; -using System.Text; using Microsoft.CodeAnalysis; namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers @@ -16,13 +13,17 @@ private static DiagnosticDescriptor Create(string id, string title,string messag return new DiagnosticDescriptor(id, title, messageFormat, category, severity, isEnabledByDefault: true, helpLinkUri: helpLink); } - public static DiagnosticDescriptor WebJobsAttributesAreNotSuppoted { get; } + public static DiagnosticDescriptor WebJobsAttributesAreNotSupported { get; } = Create(id: "AZFW0001", title: "Invalid binding attributes", messageFormat: "The attribute '{0}' is a WebJobs attribute and not supported in the .NET Worker (Isolated Process).", category: Constants.DiagnosticsCategories.Usage, severity: DiagnosticSeverity.Error); - + public static DiagnosticDescriptor AsyncVoidReturnType { get; } = Create(id: "AZFW0002", title: "Avoid async void methods", messageFormat: "Do not use void as the return type for async methods. Use Task instead.", category: Constants.DiagnosticsCategories.Usage, severity: DiagnosticSeverity.Error); + public static DiagnosticDescriptor DeferredBindingAttributeNotSupported{ get; } + = Create(id: "AZFW0003", title: "Invalid class attribute", messageFormat: "The attribute '{0}' can only be used on trigger and input binding attributes.", + category: Constants.DiagnosticsCategories.Usage, severity: DiagnosticSeverity.Error); + } } diff --git a/sdk/Sdk.Analyzers/AttributeDataExtensions.cs b/sdk/Sdk.Analyzers/Extensions/AttributeDataExtensions.cs similarity index 64% rename from sdk/Sdk.Analyzers/AttributeDataExtensions.cs rename to sdk/Sdk.Analyzers/Extensions/AttributeDataExtensions.cs index ebec79ad5..26663c774 100644 --- a/sdk/Sdk.Analyzers/AttributeDataExtensions.cs +++ b/sdk/Sdk.Analyzers/Extensions/AttributeDataExtensions.cs @@ -22,7 +22,7 @@ public static bool IsWebJobAttribute(this AttributeData attributeData) { return false; } - + foreach (var attribute in attributeAttributes) { if (string.Equals(attribute.AttributeClass?.ToDisplayString(), Constants.Types.WebJobsBindingAttribute, @@ -34,4 +34,21 @@ public static bool IsWebJobAttribute(this AttributeData attributeData) return false; } + + /// + /// Checks if an attribute is the SupportsDeferredBinding attribute. + /// + /// The attribute to check. + /// A boolean value indicating whether the attribute is a SupportsDeferredBinding attribute. + public static bool IsSupportsDeferredBindingAttribute(this AttributeData attributeData) + { + if (string.Equals(attributeData.AttributeClass?.ToDisplayString(), + Constants.Types.SupportsDeferredBindingAttribute, + StringComparison.Ordinal)) + { + return true; + } + + return false; + } } diff --git a/sdk/Sdk.Analyzers/MethodSymbolExtensions.cs b/sdk/Sdk.Analyzers/Extensions/MethodSymbolExtensions.cs similarity index 100% rename from sdk/Sdk.Analyzers/MethodSymbolExtensions.cs rename to sdk/Sdk.Analyzers/Extensions/MethodSymbolExtensions.cs diff --git a/sdk/Sdk.Analyzers/ParameterSymbolExtensions.cs b/sdk/Sdk.Analyzers/Extensions/ParameterSymbolExtensions.cs similarity index 100% rename from sdk/Sdk.Analyzers/ParameterSymbolExtensions.cs rename to sdk/Sdk.Analyzers/Extensions/ParameterSymbolExtensions.cs diff --git a/sdk/Sdk.Analyzers/TypeSymbolExtensions.cs b/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs similarity index 100% rename from sdk/Sdk.Analyzers/TypeSymbolExtensions.cs rename to sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs diff --git a/sdk/Sdk.Analyzers/WebJobsAttributesNotSupported.cs b/sdk/Sdk.Analyzers/WebJobsAttributesNotSupported.cs index 5b74f5706..0c1485ffa 100644 --- a/sdk/Sdk.Analyzers/WebJobsAttributesNotSupported.cs +++ b/sdk/Sdk.Analyzers/WebJobsAttributesNotSupported.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers [DiagnosticAnalyzer(LanguageNames.CSharp)] public class WebJobsAttributesNotSupported : DiagnosticAnalyzer { - public override ImmutableArray SupportedDiagnostics { get { return ImmutableArray.Create(DiagnosticDescriptors.WebJobsAttributesAreNotSuppoted); } } + public override ImmutableArray SupportedDiagnostics { get { return ImmutableArray.Create(DiagnosticDescriptors.WebJobsAttributesAreNotSupported); } } public override void Initialize(AnalysisContext context) { @@ -34,7 +34,7 @@ public override void Initialize(AnalysisContext context) foreach (var attribute in webjobsAttributes) { var location = Location.Create(attribute.ApplicationSyntaxReference.SyntaxTree, attribute.ApplicationSyntaxReference.Span); - c.ReportDiagnostic(Diagnostic.Create(DiagnosticDescriptors.WebJobsAttributesAreNotSuppoted, location, attribute.AttributeClass.Name)); + c.ReportDiagnostic(Diagnostic.Create(DiagnosticDescriptors.WebJobsAttributesAreNotSupported, location, attribute.AttributeClass.Name)); } } }, SymbolKind.Method); diff --git a/sdk/release_notes.md b/sdk/release_notes.md index 52e2b8857..ff8714123 100644 --- a/sdk/release_notes.md +++ b/sdk/release_notes.md @@ -8,9 +8,9 @@ - -### Microsoft.Azure.Functions.Worker.Sdk.Analyzers (delete if not updated) +### Microsoft.Azure.Functions.Worker.Sdk.Analyzers -- +- Add analyzer for SupportsDeferredBindingAttribute #1367 ### Microsoft.Azure.Functions.Worker.Sdk.Generators diff --git a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests/AsyncVoidAnalyzerTests.cs b/test/Sdk.Analyzers.Tests/AsyncVoidAnalyzerTests.cs similarity index 96% rename from test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests/AsyncVoidAnalyzerTests.cs rename to test/Sdk.Analyzers.Tests/AsyncVoidAnalyzerTests.cs index b7b2393a7..0e3a8ae7a 100644 --- a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests/AsyncVoidAnalyzerTests.cs +++ b/test/Sdk.Analyzers.Tests/AsyncVoidAnalyzerTests.cs @@ -1,5 +1,5 @@ using Xunit; -using AnalizerTest = Microsoft.CodeAnalysis.CSharp.Testing.CSharpAnalyzerTest; +using AnalyzerTest = Microsoft.CodeAnalysis.CSharp.Testing.CSharpAnalyzerTest; using AnalyzerVerifier = Microsoft.CodeAnalysis.CSharp.Testing.XUnit.AnalyzerVerifier; using CodeFixTest = Microsoft.CodeAnalysis.CSharp.Testing.CSharpCodeFixTest; using CodeFixVerifier = Microsoft.CodeAnalysis.CSharp.Testing.CSharpCodeFixVerifier; @@ -35,7 +35,7 @@ public static async void Run([QueueTrigger(""myqueue-items"")] string myQueueIte } } }"; - var test = new AnalizerTest + var test = new AnalyzerTest { ReferenceAssemblies = LoadRequiredDependencyAssemblies(), TestCode = inputCode @@ -72,7 +72,7 @@ public static async Task Run([QueueTrigger(""myqueue-items"")] string myQueueIte } } }"; - var test = new AnalizerTest + var test = new AnalyzerTest { ReferenceAssemblies = LoadRequiredDependencyAssemblies(), TestCode = inputCode @@ -80,7 +80,7 @@ public static async Task Run([QueueTrigger(""myqueue-items"")] string myQueueIte await test.RunAsync(); } - + [Fact] public async Task AnalyzerDoesNotReportForNonAsyncCode() { @@ -100,7 +100,7 @@ public static void Run([QueueTrigger(""myqueue-items"")] string myQueueItem, Fun } } }"; - var test = new AnalizerTest + var test = new AnalyzerTest { ReferenceAssemblies = LoadRequiredDependencyAssemblies(), TestCode = inputCode @@ -165,7 +165,7 @@ public static async Task Run([QueueTrigger(""myqueue-items"")] string myQueueIte test.ExpectedDiagnostics.AddRange(new[] { expectedDiagnosticResult }); await test.RunAsync(CancellationToken.None); } - + private static ReferenceAssemblies LoadRequiredDependencyAssemblies() { var referenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( diff --git a/test/Sdk.Analyzers.Tests/DeferredBindingAttributeNotSupportedTests.cs b/test/Sdk.Analyzers.Tests/DeferredBindingAttributeNotSupportedTests.cs new file mode 100644 index 000000000..dc4908f3e --- /dev/null +++ b/test/Sdk.Analyzers.Tests/DeferredBindingAttributeNotSupportedTests.cs @@ -0,0 +1,134 @@ +using Xunit; +using AnalyzerTest = Microsoft.CodeAnalysis.CSharp.Testing.CSharpAnalyzerTest; +using Verify = Microsoft.CodeAnalysis.CSharp.Testing.XUnit.AnalyzerVerifier; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis.Testing; +using System.Collections.Immutable; + +namespace Sdk.Analyzers.Tests +{ + public class DeferredBindingAttributeNotSupportedTests + { + [Fact] + public async Task TriggerBindingClass_SupportsDeferredBindingAttribute_Diagnostics_NotExpected() + { + string testCode = @" + using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + + namespace TestBindings + { + [SupportsDeferredBinding] + public sealed class BlobTriggerAttribute : TriggerBindingAttribute + { + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.12.1-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + // test.ExpectedDiagnostics is an empty collection. + + await test.RunAsync(); + } + + [Fact] + public async Task InputBindingClass_SupportsDeferredBindingAttribute_Diagnostics_NotExpected() + { + string testCode = @" + using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + + namespace TestBindings + { + [SupportsDeferredBinding] + public sealed class BlobInputAttribute : InputBindingAttribute + { + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.12.1-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + // test.ExpectedDiagnostics is an empty collection. + + await test.RunAsync(); + } + + [Fact] + public async Task OutputBindingClass_SupportsDeferredBindingAttribute_Diagnostic_Expected() + { + string testCode = @" + using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + + namespace TestBindings + { + [SupportsDeferredBinding] + public sealed class BlobOutputAttribute : OutputBindingAttribute + { + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.12.1-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + test.ExpectedDiagnostics.Add(Verify.Diagnostic() + .WithSeverity(Microsoft.CodeAnalysis.DiagnosticSeverity.Error) + .WithSpan(6, 22, 6, 45) + .WithArguments("SupportsDeferredBindingAttribute")); + + await test.RunAsync(); + } + + [Fact] + public async Task ClassWithoutBase_SupportsDeferredBindingAttribute_Diagnostic_Expected() + { + string testCode = @" + using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + + namespace TestBindings + { + [SupportsDeferredBinding] + public sealed class JustAnotherClass + { + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.12.1-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + test.ExpectedDiagnostics.Add(Verify.Diagnostic() + .WithSeverity(Microsoft.CodeAnalysis.DiagnosticSeverity.Error) + .WithSpan(6, 22, 6, 45) + .WithArguments("SupportsDeferredBindingAttribute")); + + await test.RunAsync(); + } + } +} diff --git a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj b/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj similarity index 78% rename from test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj rename to test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj index 165dbc206..424739ca9 100644 --- a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj +++ b/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj @@ -22,9 +22,9 @@ - - - + + + diff --git a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests/WebJobsAttributesNotSupportedTests.cs b/test/Sdk.Analyzers.Tests/WebJobsAttributesNotSupportedTests.cs similarity index 95% rename from test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests/WebJobsAttributesNotSupportedTests.cs rename to test/Sdk.Analyzers.Tests/WebJobsAttributesNotSupportedTests.cs index 99d83d9cb..040a29c91 100644 --- a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests/WebJobsAttributesNotSupportedTests.cs +++ b/test/Sdk.Analyzers.Tests/WebJobsAttributesNotSupportedTests.cs @@ -1,5 +1,5 @@ using Xunit; -using AnalizerTest = Microsoft.CodeAnalysis.CSharp.Testing.CSharpAnalyzerTest; +using AnalyzerTest = Microsoft.CodeAnalysis.CSharp.Testing.CSharpAnalyzerTest; using Verify = Microsoft.CodeAnalysis.CSharp.Testing.XUnit.AnalyzerVerifier; using System.Threading.Tasks; using Microsoft.CodeAnalysis.Testing; @@ -28,7 +28,7 @@ public static void Run([HttpTrigger(AuthorizationLevel.Anonymous, ""get"")] Http } } }"; - var test = new AnalizerTest + var test = new AnalyzerTest { // TODO: This needs to pull from a local source ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( @@ -43,7 +43,7 @@ public static void Run([HttpTrigger(AuthorizationLevel.Anonymous, ""get"")] Http test.ExpectedDiagnostics.Add(Verify.Diagnostic().WithSeverity(Microsoft.CodeAnalysis.DiagnosticSeverity.Error) .WithSpan(12, 105, 12, 122).WithArguments("TimerTriggerAttribute")); - + await test.RunAsync(); } @@ -67,7 +67,7 @@ public static string Run([TimerTrigger(""0 */1 * * * *"")] MyInfo myTimer) public record MyInfo(bool IsPastDue); }"; - var test = new AnalizerTest + var test = new AnalyzerTest { ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.10.0"), @@ -98,14 +98,14 @@ public static class Function1 { [Function(""Function1"")] [return: Microsoft.Azure.WebJobs.Queue(""dest-q"")] - public static string Run([TimerTrigger(""0 */1 * * * *"")] object myTimer, + public static string Run([TimerTrigger(""0 */1 * * * *"")] object myTimer, [Blob(""samples-workitems/{queueTrigger}"", FileAccess.Read)] Stream myBlob) { return ""Azure""; } } }"; - var test = new AnalizerTest + var test = new AnalyzerTest { ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.10.0"), @@ -142,7 +142,7 @@ public void Run([TimerTrigger(""0 */5 * * * *"")] object myTimer) } } }"; - var test = new AnalizerTest + var test = new AnalyzerTest { // TODO: This needs to pull from a local source ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( From 94bb624a14f3eb6e692265e45af30ab8a38414ea Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Thu, 23 Feb 2023 13:47:23 -0800 Subject: [PATCH 03/47] Update SupportsDeferredBinding diagnostic code & update docs (#1377) --- docs/analyzer-rules/AZFW0009.md | 26 ++++++++++++++++++++++ sdk/Sdk.Analyzers/DiagnosticDescriptors.cs | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 docs/analyzer-rules/AZFW0009.md diff --git a/docs/analyzer-rules/AZFW0009.md b/docs/analyzer-rules/AZFW0009.md new file mode 100644 index 000000000..4faac9a1b --- /dev/null +++ b/docs/analyzer-rules/AZFW0009.md @@ -0,0 +1,26 @@ +# AZFW0009: Invalid use of SupportsDeferredBinding attribute + +| | Value | +|-|-| +| **Rule ID** |AZFW0009| +| **Category** |[Usage]| +| **Severity** |Error| + +## Cause + +This rule is triggered when the `SupportsDeferredBinding` attribute is used on any class other +than an input (`InputBindingAttribute`) or trigger (`TriggerBindingAttribute`) binding based class. + +## Rule description + +The `SupportsDeferredBinding` attribute is used to determine if a binding supports deferred binding. +Currently, this feature is only supported for input and trigger bindings. Output bindings are not supported +and this attribute should not be used on any other class type. + +## How to fix violations + +Remove the use of the `SupportsDeferredBinding` attribute from your class. + +## When to suppress warnings + +This rule should not be suppressed because this error may prevent your Azure Functions from running. diff --git a/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs b/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs index 2a094e367..70f156f7c 100644 --- a/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs +++ b/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs @@ -22,7 +22,7 @@ private static DiagnosticDescriptor Create(string id, string title,string messag category: Constants.DiagnosticsCategories.Usage, severity: DiagnosticSeverity.Error); public static DiagnosticDescriptor DeferredBindingAttributeNotSupported{ get; } - = Create(id: "AZFW0003", title: "Invalid class attribute", messageFormat: "The attribute '{0}' can only be used on trigger and input binding attributes.", + = Create(id: "AZFW0009", title: "Invalid class attribute", messageFormat: "The attribute '{0}' can only be used on trigger and input binding attributes.", category: Constants.DiagnosticsCategories.Usage, severity: DiagnosticSeverity.Error); } From 8b400d5bbaf8cd4505b0e64019a67215e9542e71 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Tue, 13 Jun 2023 08:38:44 -0700 Subject: [PATCH 04/47] Analyzer for code suggestion for extension supported binding types (#1604) --- .../BindingTypeCodeRefactoringProvider.cs | 129 ++++++++++++++++++ sdk/Sdk.Analyzers/BindingTypeNotSupported.cs | 127 +++++++++++++++++ .../Extensions/TypeSymbolExtensions.cs | 23 +++- sdk/release_notes.md | 3 +- ...BindingTypeCodeRefactoringProviderTests.cs | 115 ++++++++++++++++ .../Sdk.Analyzers.Tests.csproj | 3 + 6 files changed, 397 insertions(+), 3 deletions(-) create mode 100644 sdk/Sdk.Analyzers/BindingTypeCodeRefactoringProvider.cs create mode 100644 sdk/Sdk.Analyzers/BindingTypeNotSupported.cs create mode 100644 test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs diff --git a/sdk/Sdk.Analyzers/BindingTypeCodeRefactoringProvider.cs b/sdk/Sdk.Analyzers/BindingTypeCodeRefactoringProvider.cs new file mode 100644 index 000000000..a009d5bc1 --- /dev/null +++ b/sdk/Sdk.Analyzers/BindingTypeCodeRefactoringProvider.cs @@ -0,0 +1,129 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Composition; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeActions; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +using Microsoft.CodeAnalysis.CodeRefactorings; + +namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers +{ + [ExportCodeRefactoringProvider(LanguageNames.CSharp, Name = nameof(BindingTypeCodeRefactoringProvider)), Shared] + public sealed class BindingTypeCodeRefactoringProvider : CodeRefactoringProvider + { + public override async Task ComputeRefactoringsAsync(CodeRefactoringContext context) + { + SyntaxNode root = await context.Document.GetSyntaxRootAsync(context.CancellationToken).ConfigureAwait(false); + + var parameters = root.DescendantNodes().OfType(); + + foreach (var parameter in parameters) + { + await AnalyzeForCodeRefactor(context, parameter); + } + } + + private async Task AnalyzeForCodeRefactor(CodeRefactoringContext context, ParameterSyntax parameter) + { + var semanticModel = await context.Document.GetSemanticModelAsync().ConfigureAwait(false); + var parameterSymbol = semanticModel.GetDeclaredSymbol(parameter); + + // Here, parameterSymbol represents a BindingAttribute parameter + foreach (var attribute in parameterSymbol.GetAttributes()) + { + var attributeType = attribute?.AttributeClass; + var inputConverterAttributes = attributeType.GetInputConverterAttributes(semanticModel.Compilation); + + if (inputConverterAttributes.Count <= 0) + { + continue; + } + + var supportedTypes = GetSupportedTypes(semanticModel, inputConverterAttributes); + + if (supportedTypes.Count <= 0) + { + continue; + } + + foreach (ITypeSymbol supportedType in supportedTypes) + { + string supportedTypeName = supportedType.GetMinimalDisplayName(semanticModel); + + // Create a code action for each potential supported type + context.RegisterRefactoring( + new SupportedBindingTypeCodeAction(context.Document, parameter, supportedTypeName)); + } + } + } + + private static List GetSupportedTypes(SemanticModel model, List inputConverterAttributes) + { + var supportedTypes = new List(); + + foreach (var inputConverterAttribute in inputConverterAttributes) + { + var converterName = inputConverterAttribute.ConstructorArguments.FirstOrDefault().Value.ToString(); + var converter = model.Compilation.GetTypeByMetadataName(converterName); + var converterAttributes = converter.GetAttributes(); + + var supportedConverterTypeAttributeType = model.Compilation.GetTypeByMetadataName(Constants.Types.SupportedConverterTypeAttribute); + + supportedTypes.AddRange(converterAttributes + .Where(a => SymbolEqualityComparer.Default.Equals(a.AttributeClass, supportedConverterTypeAttributeType)) + .Select(a => (ITypeSymbol)a.ConstructorArguments.FirstOrDefault().Value) + .ToList()); + } + + return supportedTypes; + } + + /// + /// CodeAction implementation which fixes changes async void to async Task as the return type of the method. + /// + private sealed class SupportedBindingTypeCodeAction : CodeAction + { + private readonly Document _document; + private readonly ParameterSyntax _parameterSyntax; + private readonly string _supportedType; + + internal SupportedBindingTypeCodeAction(Document document, ParameterSyntax parameterSyntax, string supportedType) + { + this._document = document; + this._parameterSyntax = parameterSyntax; + this._supportedType = supportedType; + } + + public override string Title => $"Bind to {_supportedType}"; + + /// null value is fine since we do not have more than one fix action from this code fix provider. + public override string EquivalenceKey => null; + + /// + /// Returns an updated Document where the invalid binding type is replaced with a supported type. + /// + protected override async Task GetChangedDocumentAsync(CancellationToken cancellationToken) + { + SyntaxTrivia spaceTrivia = SyntaxFactory.Whitespace(" "); + + TypeSyntax newTypeSyntax = SyntaxFactory.ParseTypeName(_supportedType); + ParameterSyntax newParameterSyntax = _parameterSyntax + .WithType(newTypeSyntax) + .WithIdentifier(_parameterSyntax.Identifier.WithLeadingTrivia(spaceTrivia)); + + SyntaxNode root = await _document.GetSyntaxRootAsync(cancellationToken).ConfigureAwait(false); + SyntaxNode newRoot = root.ReplaceNode(_parameterSyntax, newParameterSyntax); + + return _document.WithSyntaxRoot(newRoot); + } + } + } +} diff --git a/sdk/Sdk.Analyzers/BindingTypeNotSupported.cs b/sdk/Sdk.Analyzers/BindingTypeNotSupported.cs new file mode 100644 index 000000000..ac52325ce --- /dev/null +++ b/sdk/Sdk.Analyzers/BindingTypeNotSupported.cs @@ -0,0 +1,127 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers +{ + [DiagnosticAnalyzer(LanguageNames.CSharp)] + public class BindingTypeNotSupported : DiagnosticAnalyzer + { + public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(DiagnosticDescriptors.BindingTypeNotSupported); + + public override void Initialize(AnalysisContext context) + { + context.EnableConcurrentExecution(); + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.Analyze); + context.RegisterSymbolAction(AnalyzeMethod, SymbolKind.Method); + } + + private static void AnalyzeMethod(SymbolAnalysisContext context) + { + var method = (IMethodSymbol)context.Symbol; + + if (!method.IsFunction(context)) + { + return; + } + + var methodParameters = method.Parameters; + + if (method.Parameters.Length <= 0) + { + return; + } + + foreach (var parameter in methodParameters) + { + AnalyzeParameter(context, parameter); + } + } + + private static void AnalyzeParameter(SymbolAnalysisContext context, IParameterSymbol parameter) + { + foreach (var attribute in parameter.GetAttributes()) + { + var attributeType = attribute?.AttributeClass; + if (!attributeType.IsInputOrTriggerBinding()) + { + continue; + } + + var inputConverterAttributes = attributeType.GetInputConverterAttributes(context.Compilation); + if (inputConverterAttributes.Count <= 0) + { + continue; + } + + var allowConverterFallbackParameterValue = GetAllowConverterFallbackParameterValue(context, attributeType); + if (allowConverterFallbackParameterValue is bool allowFallback && allowFallback) + { + // If allowConverterFallback is true, we don't need to check for supported types + // because we don't know all of the types that are supported via the fallback + continue; + } + + var supportedTypes = GetSupportedTypes(context, inputConverterAttributes); + if (supportedTypes.Count <= 0 || supportedTypes.Contains(parameter.Type)) + { + continue; + } + + ReportDiagnostic(context, parameter, attributeType); + } + } + + private static object GetAllowConverterFallbackParameterValue(SymbolAnalysisContext context, ITypeSymbol attributeType) + { + var allowConverterFallbackAttributeType = context.Compilation.GetTypeByMetadataName(Constants.Types.AllowConverterFallbackAttribute); + var allowConverterFallbackAttribute = attributeType.GetAttributes().FirstOrDefault(attr => SymbolEqualityComparer.Default.Equals(attr.AttributeClass, allowConverterFallbackAttributeType)); + return allowConverterFallbackAttribute.ConstructorArguments.FirstOrDefault().Value; + } + + private static List GetSupportedTypes(SymbolAnalysisContext context, List inputConverterAttributes) + { + var supportedTypes = new List(); + + foreach (var inputConverterAttribute in inputConverterAttributes) + { + var converterName = inputConverterAttribute.ConstructorArguments.FirstOrDefault().Value.ToString(); + var converter = context.Compilation.GetTypeByMetadataName(converterName); + + var converterAttributes = converter.GetAttributes(); + + var supportedConverterTypeAttributeType = context.Compilation.GetTypeByMetadataName(Constants.Types.SupportedConverterTypeAttribute); + var converterHasSupportedTypeAttribute = converterAttributes.Any(a => SymbolEqualityComparer.Default.Equals(a.AttributeClass, supportedConverterTypeAttributeType)); + if (!converterHasSupportedTypeAttribute) + { + // If a converter does not have the `SupportedConverterTypeAttribute`, we don't need to check for supported types + continue; + } + + supportedTypes.AddRange(converterAttributes + .Where(a => SymbolEqualityComparer.Default.Equals(a.AttributeClass, supportedConverterTypeAttributeType)) + .SelectMany(a => a.ConstructorArguments.Select(arg => arg.Value)) + .ToList()); + } + + return supportedTypes; + } + + private static void ReportDiagnostic(SymbolAnalysisContext context, IParameterSymbol parameter, ITypeSymbol attributeType) + { + var diagnostic = Diagnostic.Create( + DiagnosticDescriptors.BindingTypeNotSupported, + parameter.Locations.First(), + parameter.Type.Name, + attributeType.Name); + + context.ReportDiagnostic(diagnostic); + } + } +} diff --git a/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs b/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs index 72dc6fed9..2633b3fad 100644 --- a/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs +++ b/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs @@ -1,10 +1,9 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; using System.Collections.Generic; using System.Linq; -using System.Text; +using System.Text.RegularExpressions; using Microsoft.CodeAnalysis; namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers @@ -38,5 +37,25 @@ internal static bool IsAssignableFrom(this ITypeSymbol targetType, ITypeSymbol s return false; } + + internal static List GetInputConverterAttributes(this ITypeSymbol attributeType, Compilation compilation) + { + var inputConverterAttributeType = compilation.GetTypeByMetadataName(Constants.Types.InputConverterAttribute); + return attributeType.GetAttributes() + .Where(attr => SymbolEqualityComparer.Default.Equals(attr.AttributeClass, inputConverterAttributeType)) + .ToList(); + } + + internal static string GetMinimalDisplayName(this ITypeSymbol type, SemanticModel semanticModel) + { + string name = type.ToMinimalDisplayString(semanticModel, 0); + + if (name.Contains("IEnumerable")) + { + name = Regex.Match(name, @"IEnumerable<[^>]+>").Value; + } + + return name; + } } } diff --git a/sdk/release_notes.md b/sdk/release_notes.md index ff8714123..ee955277f 100644 --- a/sdk/release_notes.md +++ b/sdk/release_notes.md @@ -10,7 +10,8 @@ ### Microsoft.Azure.Functions.Worker.Sdk.Analyzers -- Add analyzer for SupportsDeferredBindingAttribute #1367 +- Added an analyzer that will show a warning for types not supported by a binding attribute (#1505) +- Added an analyzer that will suggest a code refactor for all of the types supported by a binding attribute (#1604) ### Microsoft.Azure.Functions.Worker.Sdk.Generators diff --git a/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs b/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs new file mode 100644 index 000000000..ff97c9cba --- /dev/null +++ b/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs @@ -0,0 +1,115 @@ + +using System.Collections.Generic; +using System.Reflection; +using Microsoft.Azure.Functions.Worker.Sdk.Analyzers; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeRefactorings; +using RoslynTestKit; +using Xunit; + +namespace Sdk.Analyzers.Tests +{ + public class BindingTypeCodeRefactoringProviderTests : CodeRefactoringTestFixture + { + protected override string LanguageName => LanguageNames.CSharp; + + protected override CodeRefactoringProvider CreateProvider() + { + return new BindingTypeCodeRefactoringProvider(); + } + + protected override IReadOnlyCollection References => new[] + { + ReferenceSource.NetStandard2_0, + ReferenceSource.FromAssembly(Assembly.Load("System.Runtime, Version=7.0.0.0").Location), + ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Core, Version=1.13.0.0").Location), + ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.Abstractions, Version=1.2.0.0")), + + ReferenceSource.FromAssembly(Assembly.Load("Azure.Data.Tables, Version=12.8.0.0").Location), + ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.Tables, Version=1.2.0.0").Location), + + ReferenceSource.FromAssembly(Assembly.Load("Azure.Messaging.ServiceBus, Version=7.14.0.0").Location), + ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.ServiceBus, Version=5.10.0.0").Location), + }; + + [Theory] + [InlineData("TableClient", 0)] + [InlineData("TableEntity", 1)] + public void TableInput_SuggestsCodeRefactor(string supportedType, int index) + { + string testCode = @" + using System; + using Azure.Data.Tables; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([TableInput(""input"")] [|string message|]) + { + } + } + }"; + + string expectedCode = $@" + using System; + using Azure.Data.Tables; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + {{ + public static class SomeFunction + {{ + [Function(nameof(SomeFunction))] + public static void Run([TableInput(""input"")] {supportedType} message) + {{ + }} + }} + }}"; + + TestCodeRefactoring(testCode, expectedCode, index); + } + + [Theory] + [InlineData("ServiceBusReceivedMessage", 0)] + [InlineData("ServiceBusReceivedMessage[]", 1)] + public void ServiceBusTrigger_SuggestsCodeRefactor(string supportedType, int index) + { + string testCode = @" + using System; + using Azure.Messaging.ServiceBus; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([ServiceBusTrigger(""input"")] [|string message|]) + { + } + } + }"; + + string expectedCode = $@" + using System; + using Azure.Messaging.ServiceBus; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + {{ + public static class SomeFunction + {{ + [Function(nameof(SomeFunction))] + public static void Run([ServiceBusTrigger(""input"")] {supportedType} message) + {{ + }} + }} + }}"; + + TestCodeRefactoring(testCode, expectedCode, index); + } + } +} diff --git a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj b/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj index 424739ca9..5c84f5ced 100644 --- a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj +++ b/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj @@ -8,6 +8,7 @@ + all @@ -23,6 +24,8 @@ + + From ccbcba6ad1e0b4b09c23ca9bc0ff301dcdb04155 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 7 Jun 2023 13:49:21 -0700 Subject: [PATCH 05/47] [Analyzer] Warning for extension constrained types (#1505) --- docs/analyzer-rules/AZFW0010.md | 42 +++++ sdk/Sdk.Analyzers/AsyncVoidAnalyzer.cs | 6 +- sdk/Sdk.Analyzers/AsyncVoidCodeFixProvider.cs | 2 +- sdk/Sdk.Analyzers/Constants.cs | 3 + .../DeferredBindingAttributeNotSupported.cs | 18 +- .../Extensions/NamedSymbolExtensions.cs | 24 +++ sdk/Sdk.Analyzers/Sdk.Analyzers.csproj | 6 +- .../WebJobsAttributesNotSupported.cs | 2 +- sdk/release_notes.md | 3 +- .../SdkTests.csproj | 4 +- .../BindingTypeNotSupportedTests.cs | 159 ++++++++++++++++++ 11 files changed, 242 insertions(+), 27 deletions(-) create mode 100644 docs/analyzer-rules/AZFW0010.md create mode 100644 sdk/Sdk.Analyzers/Extensions/NamedSymbolExtensions.cs create mode 100644 test/Sdk.Analyzers.Tests/BindingTypeNotSupportedTests.cs diff --git a/docs/analyzer-rules/AZFW0010.md b/docs/analyzer-rules/AZFW0010.md new file mode 100644 index 000000000..4939c75d6 --- /dev/null +++ b/docs/analyzer-rules/AZFW0010.md @@ -0,0 +1,42 @@ +# AZFW0010: Invalid binding type + +| | Value | +|-|-| +| **Rule ID** |AZFW00010| +| **Category** |[Usage]| +| **Severity** |Warning| + +## Cause + +This rule is triggered when a function is binding to a type that is not supported +by the binding attribute being used. + +## Rule description + +Some bindings advertise the types that they support. If a binding advertises supported types +and does not support falling back to built-in converters, then this rule will will be flagged. + +For example if you're binding to a `QueueTrigger` and the only supported type is `string` and +your function is binding to a `bool`, then you will see this warning. + +## How to fix violations + +Change the binding type to a type that is supported by the binding your function is using. + +### Supported Types + +You can refer to the [public documentation](https://learn.microsoft.com/azure/azure-functions/functions-bindings-storage-blob?tabs=in-process%2Cextensionv5%2Cextensionv3&pivots=programming-language-csharp) to see which types are supported. + +| Binding | Supported Types | +| ------- | --------------- | +| BlobTrigger | POCO, String, Stream, Byte[], BlobClient*, BlobContainerClient | +| BlobInput | POCO, IEnumerable*, String, IEnumberable, Stream, IEnumberable, Byte[], BlobClient, IEnumerable, BlobContainerClient | +| CosmosDBTrigger | POCO, IEnumerable | +| CosmosDBInput | POCO, IEnumerable, CosmosClient, Database, Container | +| EventGridTrigger | String, IEnumerable, CloudEvent, IEnumerable, EventGridEvent, IEnumerable | +| EventHubTrigger | POCO, POCO[], String, String[] | +| QueueTrigger | POCO, String, BinaryData, QueueMessage | +| ServiceBusTrigger | String, String[], ServiceBusReceivedMessage, ServiceBusReceivedMessage[] | +| TableInput | POCO, IEnumerable, TableEntity, IEnumerable, TableClient | + +> \* Where `BlobClient` is supported AppendBlobClient, BaseBlobClient, BlockBlobClient and PageBlobClient are also supported diff --git a/sdk/Sdk.Analyzers/AsyncVoidAnalyzer.cs b/sdk/Sdk.Analyzers/AsyncVoidAnalyzer.cs index 9a8c35bc6..43e56f676 100644 --- a/sdk/Sdk.Analyzers/AsyncVoidAnalyzer.cs +++ b/sdk/Sdk.Analyzers/AsyncVoidAnalyzer.cs @@ -9,8 +9,8 @@ namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers { [DiagnosticAnalyzer(LanguageNames.CSharp)] public sealed class AsyncVoidAnalyzer : DiagnosticAnalyzer - { - public override ImmutableArray SupportedDiagnostics => ImmutableArray.Create(DiagnosticDescriptors.AsyncVoidReturnType); + { + public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(DiagnosticDescriptors.AsyncVoidReturnType); public override void Initialize(AnalysisContext context) { @@ -27,7 +27,7 @@ private static void AnalyzeMethod(SymbolAnalysisContext symbolAnalysisContext) if (symbol.IsAsync && symbol.ReturnsVoid) { // This symbol is a method symbol and will have only one item in Locations property. - var location = symbol.Locations[0]; + var location = symbol.Locations[0]; var diagnostic = Diagnostic.Create(DiagnosticDescriptors.AsyncVoidReturnType, location); symbolAnalysisContext.ReportDiagnostic(diagnostic); } diff --git a/sdk/Sdk.Analyzers/AsyncVoidCodeFixProvider.cs b/sdk/Sdk.Analyzers/AsyncVoidCodeFixProvider.cs index e91bfc4f2..1c07cbf8e 100644 --- a/sdk/Sdk.Analyzers/AsyncVoidCodeFixProvider.cs +++ b/sdk/Sdk.Analyzers/AsyncVoidCodeFixProvider.cs @@ -27,7 +27,7 @@ public override Task RegisterCodeFixesAsync(CodeFixContext context) { Diagnostic diagnostic = context.Diagnostics.First(); context.RegisterCodeFix(new VoidToTaskCodeAction(context.Document, diagnostic), diagnostic); - + return Task.CompletedTask; } diff --git a/sdk/Sdk.Analyzers/Constants.cs b/sdk/Sdk.Analyzers/Constants.cs index b526a6363..2f084c8e9 100644 --- a/sdk/Sdk.Analyzers/Constants.cs +++ b/sdk/Sdk.Analyzers/Constants.cs @@ -12,6 +12,9 @@ internal static class Types public const string SupportsDeferredBindingAttribute = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.SupportsDeferredBindingAttribute"; public const string InputBindingAttribute = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.InputBindingAttribute"; public const string TriggerBindingAttribute = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.TriggerBindingAttribute"; + public const string InputConverterAttribute = "Microsoft.Azure.Functions.Worker.Converters.InputConverterAttribute"; + public const string AllowConverterFallbackAttribute = "Microsoft.Azure.Functions.Worker.Converters.AllowConverterFallbackAttribute"; + public const string SupportedConverterTypeAttribute = "Microsoft.Azure.Functions.Worker.Converters.SupportedConverterTypeAttribute"; // System types internal const string TaskType = "System.Threading.Tasks.Task"; diff --git a/sdk/Sdk.Analyzers/DeferredBindingAttributeNotSupported.cs b/sdk/Sdk.Analyzers/DeferredBindingAttributeNotSupported.cs index 459e1b2ce..20e428386 100644 --- a/sdk/Sdk.Analyzers/DeferredBindingAttributeNotSupported.cs +++ b/sdk/Sdk.Analyzers/DeferredBindingAttributeNotSupported.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; using System.Collections.Immutable; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.Diagnostics; @@ -11,7 +10,7 @@ namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers [DiagnosticAnalyzer(LanguageNames.CSharp)] public class DeferredBindingAttributeNotSupported : DiagnosticAnalyzer { - public override ImmutableArray SupportedDiagnostics { get { return ImmutableArray.Create(DiagnosticDescriptors.DeferredBindingAttributeNotSupported); } } + public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(DiagnosticDescriptors.DeferredBindingAttributeNotSupported); public override void Initialize(AnalysisContext context) { @@ -33,7 +32,7 @@ private static void AnalyzeMethod(SymbolAnalysisContext symbolAnalysisContext) foreach (var attribute in attributes) { - if (attribute.IsSupportsDeferredBindingAttribute() && !IsInputOrTriggerBinding(symbol)) + if (attribute.IsSupportsDeferredBindingAttribute() && !symbol.IsInputOrTriggerBinding()) { var location = Location.Create(attribute.ApplicationSyntaxReference.SyntaxTree, attribute.ApplicationSyntaxReference.Span); var diagnostic = Diagnostic.Create(DiagnosticDescriptors.DeferredBindingAttributeNotSupported, location, attribute.AttributeClass.Name); @@ -41,18 +40,5 @@ private static void AnalyzeMethod(SymbolAnalysisContext symbolAnalysisContext) } } } - - private static bool IsInputOrTriggerBinding(INamedTypeSymbol symbol) - { - var baseType = symbol.BaseType?.ToDisplayString(); - - if (string.Equals(baseType,Constants.Types.InputBindingAttribute, StringComparison.Ordinal) - || string.Equals(baseType,Constants.Types.TriggerBindingAttribute, StringComparison.Ordinal)) - { - return true; - } - - return false; - } } } diff --git a/sdk/Sdk.Analyzers/Extensions/NamedSymbolExtensions.cs b/sdk/Sdk.Analyzers/Extensions/NamedSymbolExtensions.cs new file mode 100644 index 000000000..973c3a65e --- /dev/null +++ b/sdk/Sdk.Analyzers/Extensions/NamedSymbolExtensions.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.CodeAnalysis; + +namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers +{ + internal static class NamedSymbolExtensions + { + internal static bool IsInputOrTriggerBinding(this INamedTypeSymbol symbol) + { + var baseType = symbol.BaseType?.ToDisplayString(); + + if (string.Equals(baseType,Constants.Types.InputBindingAttribute, StringComparison.Ordinal) + || string.Equals(baseType,Constants.Types.TriggerBindingAttribute, StringComparison.Ordinal)) + { + return true; + } + + return false; + } + } +} diff --git a/sdk/Sdk.Analyzers/Sdk.Analyzers.csproj b/sdk/Sdk.Analyzers/Sdk.Analyzers.csproj index c268cf682..4630f3716 100644 --- a/sdk/Sdk.Analyzers/Sdk.Analyzers.csproj +++ b/sdk/Sdk.Analyzers/Sdk.Analyzers.csproj @@ -1,8 +1,8 @@  - 1 - 2 + 2 + 0 Library true false @@ -29,7 +29,7 @@ - + diff --git a/sdk/Sdk.Analyzers/WebJobsAttributesNotSupported.cs b/sdk/Sdk.Analyzers/WebJobsAttributesNotSupported.cs index 0c1485ffa..250bb2d71 100644 --- a/sdk/Sdk.Analyzers/WebJobsAttributesNotSupported.cs +++ b/sdk/Sdk.Analyzers/WebJobsAttributesNotSupported.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers [DiagnosticAnalyzer(LanguageNames.CSharp)] public class WebJobsAttributesNotSupported : DiagnosticAnalyzer { - public override ImmutableArray SupportedDiagnostics { get { return ImmutableArray.Create(DiagnosticDescriptors.WebJobsAttributesAreNotSupported); } } + public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(DiagnosticDescriptors.WebJobsAttributesAreNotSupported); public override void Initialize(AnalysisContext context) { diff --git a/sdk/release_notes.md b/sdk/release_notes.md index ee955277f..01dd430aa 100644 --- a/sdk/release_notes.md +++ b/sdk/release_notes.md @@ -8,8 +8,9 @@ - -### Microsoft.Azure.Functions.Worker.Sdk.Analyzers +### Microsoft.Azure.Functions.Worker.Sdk.Analyzers 1.2.0 +- Add analyzer for SupportsDeferredBindingAttribute (#1367) - Added an analyzer that will show a warning for types not supported by a binding attribute (#1505) - Added an analyzer that will suggest a code refactor for all of the types supported by a binding attribute (#1604) diff --git a/test/FunctionMetadataGeneratorTests/SdkTests.csproj b/test/FunctionMetadataGeneratorTests/SdkTests.csproj index 16a13f46f..bd05713cc 100644 --- a/test/FunctionMetadataGeneratorTests/SdkTests.csproj +++ b/test/FunctionMetadataGeneratorTests/SdkTests.csproj @@ -9,12 +9,12 @@ $(NoWarn);NU1608;NU1701 - + - + all diff --git a/test/Sdk.Analyzers.Tests/BindingTypeNotSupportedTests.cs b/test/Sdk.Analyzers.Tests/BindingTypeNotSupportedTests.cs new file mode 100644 index 000000000..c8ef7c254 --- /dev/null +++ b/test/Sdk.Analyzers.Tests/BindingTypeNotSupportedTests.cs @@ -0,0 +1,159 @@ +using Xunit; +using AnalyzerTest = Microsoft.CodeAnalysis.CSharp.Testing.CSharpAnalyzerTest; +using Verify = Microsoft.CodeAnalysis.CSharp.Testing.XUnit.AnalyzerVerifier; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis.Testing; +using System.Collections.Immutable; + +namespace Sdk.Analyzers.Tests +{ + public class BindingTypeNotSupportedTests + { + [Fact] + public async Task TestAttribute_ValidBindingType_DiagnosticsNotExpected() + { + string testCode = @" + using System; + using Microsoft.Azure.Functions.Worker; + using Microsoft.Azure.Functions.Worker.Core; + using Microsoft.Azure.Functions.Worker.Converters; + using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + + namespace Microsoft.Azure.Functions.Worker + { + [ConverterFallbackBehavior(ConverterFallbackBehavior.Disallow)] + [InputConverter(typeof(TestConverter))] + public class TestTriggerAttribute : TriggerBindingAttribute + { + } + + [SupportsDeferredBinding] + [SupportedTargetType(typeof(string))] + [SupportedTargetType(typeof(bool))] + public class TestConverter + { + } + } + + namespace FunctionApp + { + public class SomeFunction + { + [Function(nameof(SomeFunction))] + public void Run([TestTrigger()] string message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.18.0"))), + TestCode = testCode + }; + + // test.ExpectedDiagnostics is an empty collection. + + await test.RunAsync(); + } + + [Fact] + public async Task TestAttribute_ValidBindingType_WithoutDeferredBinding_DiagnosticsNotExpected() + { + string testCode = @" + using System; + using Microsoft.Azure.Functions.Worker; + using Microsoft.Azure.Functions.Worker.Core; + using Microsoft.Azure.Functions.Worker.Converters; + using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + + namespace Microsoft.Azure.Functions.Worker + { + [ConverterFallbackBehavior(ConverterFallbackBehavior.Disallow)] + [InputConverter(typeof(TestConverter))] + public class TestTriggerAttribute : TriggerBindingAttribute + { + } + + [SupportedTargetType(typeof(string))] + [SupportedTargetType(typeof(bool))] + public class TestConverter + { + } + } + + namespace FunctionApp + { + public class SomeFunction + { + [Function(nameof(SomeFunction))] + public void Run([TestTrigger()] string message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.18.0"))), + TestCode = testCode + }; + + // test.ExpectedDiagnostics is an empty collection. + + await test.RunAsync(); + } + + [Fact] + public async Task TestAttribute_InvalidBindingType_DiagnosticsExpected() + { + string testCode = @" + using System; + using Microsoft.Azure.Functions.Worker; + using Microsoft.Azure.Functions.Worker.Core; + using Microsoft.Azure.Functions.Worker.Converters; + using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + + namespace Microsoft.Azure.Functions.Worker + { + [ConverterFallbackBehavior(ConverterFallbackBehavior.Disallow)] + [InputConverter(typeof(TestConverter))] + public class TestTriggerAttribute : TriggerBindingAttribute + { + } + + [SupportsDeferredBinding] + [SupportedTargetType(typeof(string))] + [SupportedTargetType(typeof(bool))] + public class TestConverter + { + } + } + + namespace FunctionApp + { + public class SomeFunction + { + [Function(nameof(SomeFunction))] + public void Run([TestTrigger()] BinaryData message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.18.0"))), + TestCode = testCode + }; + + test.ExpectedDiagnostics.Add(Verify.Diagnostic().WithSeverity(Microsoft.CodeAnalysis.DiagnosticSeverity.Warning) + .WithSpan(29, 68, 29, 75).WithArguments("BinaryData", "TestTriggerAttribute")); + + await test.RunAsync(); + } + } +} From 318daafd1a348e8d0d3a02bfc6995f5388273b9e Mon Sep 17 00:00:00 2001 From: Surgupta Date: Tue, 20 Jun 2023 22:21:37 -0700 Subject: [PATCH 06/47] Analyzer to check if a blob container is not binding to an iterable type parameter (#1645) --- sdk/Sdk.Analyzers/DiagnosticDescriptors.cs | 7 + .../Extensions/TypeSymbolExtensions.cs | 81 ++++++ ...BindingTypeExpectedForBlobContainerPath.cs | 89 ++++++ ...ngTypeExpectedForBlobContainerPathTests.cs | 265 ++++++++++++++++++ 4 files changed, 442 insertions(+) create mode 100644 sdk/Sdk.Analyzers/IterableBindingTypeExpectedForBlobContainerPath.cs create mode 100644 test/Sdk.Analyzers.Tests/IterableBindingTypeExpectedForBlobContainerPathTests.cs diff --git a/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs b/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs index 70f156f7c..7762bb268 100644 --- a/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs +++ b/sdk/Sdk.Analyzers/DiagnosticDescriptors.cs @@ -25,5 +25,12 @@ private static DiagnosticDescriptor Create(string id, string title,string messag = Create(id: "AZFW0009", title: "Invalid class attribute", messageFormat: "The attribute '{0}' can only be used on trigger and input binding attributes.", category: Constants.DiagnosticsCategories.Usage, severity: DiagnosticSeverity.Error); + public static DiagnosticDescriptor BindingTypeNotSupported{ get; } + = Create(id: "AZFW0010", title: "Invalid binding type", messageFormat: "The binding type '{0}' is not supported by '{1}'.", + category: Constants.DiagnosticsCategories.Usage, severity: DiagnosticSeverity.Warning); + + public static DiagnosticDescriptor IterableBindingTypeExpectedForBlobContainer { get; } + = Create(id: "AZFW0011", title: "Invalid binding type", messageFormat: "The binding type '{0}' must be iterable for container path.", + category: Constants.DiagnosticsCategories.Usage, severity: DiagnosticSeverity.Error); } } diff --git a/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs b/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs index 2633b3fad..246538acb 100644 --- a/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs +++ b/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs @@ -1,10 +1,13 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System.Collections; +using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers { @@ -57,5 +60,83 @@ internal static string GetMinimalDisplayName(this ITypeSymbol type, SemanticMode return name; } + + internal static bool IsIterableType(this ITypeSymbol typeSymbol, SymbolAnalysisContext context) + { + bool isArrayType = false; + + if (typeSymbol is IArrayTypeSymbol) + { + if (string.Equals(typeSymbol.ToString(), typeof(byte[]).Name, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + isArrayType = true; + } + + bool IsIEnumerableTType = typeSymbol.IsOrImplementsOrDerivesFrom(context.Compilation.GetTypeByMetadataName(typeof(IEnumerable<>).FullName)!); + + var IsIEnumerableType = typeSymbol.IsOrImplementsOrDerivesFrom(context.Compilation.GetTypeByMetadataName(typeof(IEnumerable).FullName)!); + + return IsIEnumerableTType || IsIEnumerableType || isArrayType; + } + + internal static bool IsOrImplementsOrDerivesFrom(this ITypeSymbol symbol, ITypeSymbol? other) + { + return symbol.IsOrImplements(other) || symbol.IsOrDerivedFrom(other); + } + + internal static bool IsOrDerivedFrom(this ITypeSymbol symbol, ITypeSymbol? other) + { + if (other is null) + { + return false; + } + + var current = symbol; + + while (current != null) + { + if (SymbolEqualityComparer.Default.Equals(current, other) || SymbolEqualityComparer.Default.Equals(current.OriginalDefinition, other)) + { + return true; + } + + current = current.BaseType; + } + + return false; + } + + internal static bool IsOrImplements(this ITypeSymbol symbol, ITypeSymbol? other) + { + if (other is null) + { + return false; + } + + if (symbol.Name == typeof(string).Name) + { + return false; + } + + var current = symbol; + + while (current != null) + { + foreach (var member in current.Interfaces) + { + if (IsOrDerivedFrom(member, other)) + { + return true; + } + } + + current = current.BaseType; + } + + return false; + } } } diff --git a/sdk/Sdk.Analyzers/IterableBindingTypeExpectedForBlobContainerPath.cs b/sdk/Sdk.Analyzers/IterableBindingTypeExpectedForBlobContainerPath.cs new file mode 100644 index 000000000..7483d0da5 --- /dev/null +++ b/sdk/Sdk.Analyzers/IterableBindingTypeExpectedForBlobContainerPath.cs @@ -0,0 +1,89 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers +{ + [DiagnosticAnalyzer(LanguageNames.CSharp)] + public class IterableBindingTypeExpectedForBlobContainerPath : DiagnosticAnalyzer + { + public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(DiagnosticDescriptors.IterableBindingTypeExpectedForBlobContainer); + + private const string BlobInputBindingAttribute = "Microsoft.Azure.Functions.Worker.BlobInputAttribute"; + + public override void Initialize(AnalysisContext context) + { + context.EnableConcurrentExecution(); + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.Analyze); + context.RegisterSymbolAction(AnalyzeMethod, SymbolKind.Method); + } + + private static void AnalyzeMethod(SymbolAnalysisContext context) + { + var method = (IMethodSymbol)context.Symbol; + + if (!method.IsFunction(context)) + { + return; + } + + var methodParameters = method.Parameters; + + if (method.Parameters.Length <= 0) + { + return; + } + + foreach (var parameter in methodParameters) + { + AnalyzeParameter(context, parameter); + } + } + + private static void AnalyzeParameter(SymbolAnalysisContext context, IParameterSymbol parameter) + { + ITypeSymbol parameterType = parameter.Type; + + foreach (AttributeData attribute in parameter.GetAttributes()) + { + var attributeType = attribute?.AttributeClass; + + if (!IsBlobInputBinding(attributeType)) + { + continue; + } + + foreach (var arg in attribute.ConstructorArguments) + { + if (arg.Type.Name == typeof(string).Name) + { + string path = arg.Value.ToString(); + + if (path.Split('/').Length < 2 && !parameterType.IsIterableType(context)) + { + var diagnostic = Diagnostic.Create(DiagnosticDescriptors.IterableBindingTypeExpectedForBlobContainer, parameter.Locations.First(), parameterType); + context.ReportDiagnostic(diagnostic); + } + } + } + } + } + + internal static bool IsBlobInputBinding(INamedTypeSymbol symbol) + { + var baseType = symbol.ToDisplayString(); + + if (string.Equals(baseType, BlobInputBindingAttribute, StringComparison.Ordinal)) + { + return true; + } + + return false; + } + } +} diff --git a/test/Sdk.Analyzers.Tests/IterableBindingTypeExpectedForBlobContainerPathTests.cs b/test/Sdk.Analyzers.Tests/IterableBindingTypeExpectedForBlobContainerPathTests.cs new file mode 100644 index 000000000..7c2d7aa45 --- /dev/null +++ b/test/Sdk.Analyzers.Tests/IterableBindingTypeExpectedForBlobContainerPathTests.cs @@ -0,0 +1,265 @@ +using Xunit; +using AnalyzerTest = Microsoft.CodeAnalysis.CSharp.Testing.CSharpAnalyzerTest; +using Verify = Microsoft.CodeAnalysis.CSharp.Testing.XUnit.AnalyzerVerifier; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis.Testing; +using System.Collections.Immutable; + +namespace Sdk.Analyzers.Tests +{ + public class IterableBindingTypeExpectedForBlobContainerPathTests + { + [Fact] + public async Task BlobInputAttribute_String_Diagnostics_Expected() + { + string testCode = @" + using System; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([BlobInput(""input"")] string message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.15.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs", "5.1.1-preview2"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + test.ExpectedDiagnostics.Add(Verify.Diagnostic() + .WithSeverity(Microsoft.CodeAnalysis.DiagnosticSeverity.Error) + .WithSpan(10, 76, 10, 83) + .WithArguments("string")); + + await test.RunAsync(); + } + + [Fact] + public async Task BlobInputAttribute_Stream_Diagnostics_Expected() + { + string testCode = @" + using System; + using System.IO; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([BlobInput(""input"")] Stream message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.15.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs", "5.1.1-preview2"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + test.ExpectedDiagnostics.Add(Verify.Diagnostic() + .WithSeverity(Microsoft.CodeAnalysis.DiagnosticSeverity.Error) + .WithSpan(11, 76, 11, 83) + .WithArguments("System.IO.Stream")); + + await test.RunAsync(); + } + + [Fact] + public async Task BlobInputAttribute_ByteArray_Diagnostics_Expected() + { + string testCode = @" + using System; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([BlobInput(""input"")] byte[] message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.15.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs", "5.1.1-preview2"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + test.ExpectedDiagnostics.Add(Verify.Diagnostic() + .WithSeverity(Microsoft.CodeAnalysis.DiagnosticSeverity.Error) + .WithSpan(10, 76, 10, 83) + .WithArguments("byte[]")); + + await test.RunAsync(); + } + + + [Fact] + public async Task BlobInputAttribute_StringArray_Diagnostics_NotExpected() + { + string testCode = @" + using System; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([BlobInput(""input"")] string[] message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.15.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs", "5.1.1-preview2"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + // test.ExpectedDiagnostics is an empty collection. + + await test.RunAsync(); + } + + [Fact] + public async Task BlobInputAttribute_StreamEnumerable_Diagnostics_NotExpected() + { + string testCode = @" + using System; + using System.Collections.Generic; + using System.IO; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([BlobInput(""input"")] IEnumerable message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.15.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs", "5.1.1-preview2"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + // test.ExpectedDiagnostics is an empty collection. + + await test.RunAsync(); + } + + [Fact] + public async Task BlobInputAttribute_StringEnumerable_Diagnostics_NotExpected() + { + string testCode = @" + using System; + using System.Collections.Generic; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([BlobInput(""input"")] IEnumerable message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.15.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs", "5.1.1-preview2"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + // test.ExpectedDiagnostics is an empty collection. + + await test.RunAsync(); + } + + [Fact] + public async Task BlobInputAttribute_ArrayOfByteArray_Diagnostics_NotExpected() + { + string testCode = @" + using System; + using System.Collections.Generic; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([BlobInput(""input"")] byte[][] message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.15.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs", "5.1.1-preview2"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + // test.ExpectedDiagnostics is an empty collection. + + await test.RunAsync(); + } + } +} From d8c0f2521e6c4a7b4b30bf47747ffd56fe1d6ded Mon Sep 17 00:00:00 2001 From: Surgupta Date: Mon, 26 Jun 2023 16:35:58 -0700 Subject: [PATCH 07/47] Adding doc for analyzer rule AZFW00011 (#1689) * Adding doc for analyzer rule AZFW00011 --- docs/analyzer-rules/AZFW0011.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docs/analyzer-rules/AZFW0011.md diff --git a/docs/analyzer-rules/AZFW0011.md b/docs/analyzer-rules/AZFW0011.md new file mode 100644 index 000000000..9825203dd --- /dev/null +++ b/docs/analyzer-rules/AZFW0011.md @@ -0,0 +1,27 @@ +# AZFW0011: Invalid binding type + +| | Value | +|-|-| +| **Rule ID** |AZFW00011| +| **Category** |[Usage]| +| **Severity** |Error| + +## Cause + +This rule is triggered when a function is binding to a non-iterable type for a blob container path. + +## Rule description + +When using the `BlobInputAttribute` with a container path, the target parameter must be of iterable type such as `IEnumerable`. + +Example, if your function uses `BlobInput("")` and it is binding to a `string`, this rule will be violated. + + +## How to fix violations + +Change the binding type to an iterable type such as `IEnumerable` or provide blob path `container/blob` instead of a container path when using a non-iterable binding type is desired. + + +### Supported Types + +You can refer to the [public documentation](https://learn.microsoft.com/azure/azure-functions/functions-bindings-storage-blob?tabs=in-process%2Cextensionv5%2Cextensionv3&pivots=programming-language-csharp) to see which types are supported for blob container. From eb710d9eec99ae991a010474768636fa44c6ad42 Mon Sep 17 00:00:00 2001 From: Surgupta Date: Tue, 27 Jun 2023 17:42:04 -0700 Subject: [PATCH 08/47] Adding BlobContainerClient scenario in analyzer (#1690) --- docs/analyzer-rules/AZFW0011.md | 2 +- ...BindingTypeExpectedForBlobContainerPath.cs | 5 ++- ...ngTypeExpectedForBlobContainerPathTests.cs | 36 +++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/analyzer-rules/AZFW0011.md b/docs/analyzer-rules/AZFW0011.md index 9825203dd..7c2f8ebef 100644 --- a/docs/analyzer-rules/AZFW0011.md +++ b/docs/analyzer-rules/AZFW0011.md @@ -12,7 +12,7 @@ This rule is triggered when a function is binding to a non-iterable type for a b ## Rule description -When using the `BlobInputAttribute` with a container path, the target parameter must be of iterable type such as `IEnumerable`. +When using the `BlobInputAttribute` with a container path, the target parameter must be of iterable type such as `IEnumerable` except when binding to `BlobContainerClient`. Example, if your function uses `BlobInput("")` and it is binding to a `string`, this rule will be violated. diff --git a/sdk/Sdk.Analyzers/IterableBindingTypeExpectedForBlobContainerPath.cs b/sdk/Sdk.Analyzers/IterableBindingTypeExpectedForBlobContainerPath.cs index 7483d0da5..7a3cf0813 100644 --- a/sdk/Sdk.Analyzers/IterableBindingTypeExpectedForBlobContainerPath.cs +++ b/sdk/Sdk.Analyzers/IterableBindingTypeExpectedForBlobContainerPath.cs @@ -15,6 +15,7 @@ public class IterableBindingTypeExpectedForBlobContainerPath : DiagnosticAnalyze public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(DiagnosticDescriptors.IterableBindingTypeExpectedForBlobContainer); private const string BlobInputBindingAttribute = "Microsoft.Azure.Functions.Worker.BlobInputAttribute"; + private const string BlobContainerClientType = "Azure.Storage.Blobs.BlobContainerClient"; public override void Initialize(AnalysisContext context) { @@ -64,7 +65,9 @@ private static void AnalyzeParameter(SymbolAnalysisContext context, IParameterSy { string path = arg.Value.ToString(); - if (path.Split('/').Length < 2 && !parameterType.IsIterableType(context)) + if (path.Split('/').Length < 2 + && !parameterType.IsIterableType(context) + && !string.Equals(parameterType.ToDisplayString(), BlobContainerClientType, StringComparison.Ordinal)) { var diagnostic = Diagnostic.Create(DiagnosticDescriptors.IterableBindingTypeExpectedForBlobContainer, parameter.Locations.First(), parameterType); context.ReportDiagnostic(diagnostic); diff --git a/test/Sdk.Analyzers.Tests/IterableBindingTypeExpectedForBlobContainerPathTests.cs b/test/Sdk.Analyzers.Tests/IterableBindingTypeExpectedForBlobContainerPathTests.cs index 7c2d7aa45..280324cf7 100644 --- a/test/Sdk.Analyzers.Tests/IterableBindingTypeExpectedForBlobContainerPathTests.cs +++ b/test/Sdk.Analyzers.Tests/IterableBindingTypeExpectedForBlobContainerPathTests.cs @@ -261,5 +261,41 @@ public static void Run([BlobInput(""input"")] byte[][] message) await test.RunAsync(); } + + [Fact] + public async Task BlobInputAttribute_BlobContainerClient_Diagnostics_NotExpected() + { + string testCode = @" + using System; + using System.Collections.Generic; + using Microsoft.Azure.Functions.Worker; + using Azure.Storage.Blobs; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([BlobInput(""input"")] BlobContainerClient message) + { + } + } + }"; + + var test = new AnalyzerTest + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net50.WithPackages(ImmutableArray.Create( + new PackageIdentity("Microsoft.Azure.Functions.Worker", "1.15.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Sdk", "1.9.0-preview1"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs", "5.1.1-preview2"), + new PackageIdentity("Microsoft.Azure.Functions.Worker.Extensions.Abstractions", "1.2.0-preview1"))), + + TestCode = testCode + }; + + // test.ExpectedDiagnostics is an empty collection. + + await test.RunAsync(); + } } } From cb03a976d1407456150627e4d71cb84d7f0ce290 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 12 Jul 2023 14:41:42 -0700 Subject: [PATCH 09/47] Update analyzer and fix tests (#1741) --- .../BindingTypeCodeRefactoringProvider.cs | 4 ++-- sdk/Sdk.Analyzers/BindingTypeNotSupported.cs | 24 ++++++++++--------- sdk/Sdk.Analyzers/Constants.cs | 4 ++-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/sdk/Sdk.Analyzers/BindingTypeCodeRefactoringProvider.cs b/sdk/Sdk.Analyzers/BindingTypeCodeRefactoringProvider.cs index a009d5bc1..44a3de3f7 100644 --- a/sdk/Sdk.Analyzers/BindingTypeCodeRefactoringProvider.cs +++ b/sdk/Sdk.Analyzers/BindingTypeCodeRefactoringProvider.cs @@ -75,10 +75,10 @@ private static List GetSupportedTypes(SemanticModel model, List SymbolEqualityComparer.Default.Equals(a.AttributeClass, supportedConverterTypeAttributeType)) + .Where(a => SymbolEqualityComparer.Default.Equals(a.AttributeClass, supportedTargetTypeAttributeType)) .Select(a => (ITypeSymbol)a.ConstructorArguments.FirstOrDefault().Value) .ToList()); } diff --git a/sdk/Sdk.Analyzers/BindingTypeNotSupported.cs b/sdk/Sdk.Analyzers/BindingTypeNotSupported.cs index ac52325ce..531fafa4d 100644 --- a/sdk/Sdk.Analyzers/BindingTypeNotSupported.cs +++ b/sdk/Sdk.Analyzers/BindingTypeNotSupported.cs @@ -13,6 +13,8 @@ namespace Microsoft.Azure.Functions.Worker.Sdk.Analyzers [DiagnosticAnalyzer(LanguageNames.CSharp)] public class BindingTypeNotSupported : DiagnosticAnalyzer { + private const int ConverterFallbackBehaviorDefaultValue = 0; + public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(DiagnosticDescriptors.BindingTypeNotSupported); public override void Initialize(AnalysisContext context) @@ -60,10 +62,10 @@ private static void AnalyzeParameter(SymbolAnalysisContext context, IParameterSy continue; } - var allowConverterFallbackParameterValue = GetAllowConverterFallbackParameterValue(context, attributeType); - if (allowConverterFallbackParameterValue is bool allowFallback && allowFallback) + var converterFallbackBehaviorParameterValue = (int)GetConverterFallbackBehaviorParameterValue(context, attributeType); + if (converterFallbackBehaviorParameterValue == ConverterFallbackBehaviorDefaultValue) { - // If allowConverterFallback is true, we don't need to check for supported types + // If the ConverterFallbackBehavior is Allow or Default (enum value 0), we don't need to check for supported types // because we don't know all of the types that are supported via the fallback continue; } @@ -78,11 +80,11 @@ private static void AnalyzeParameter(SymbolAnalysisContext context, IParameterSy } } - private static object GetAllowConverterFallbackParameterValue(SymbolAnalysisContext context, ITypeSymbol attributeType) + private static object GetConverterFallbackBehaviorParameterValue(SymbolAnalysisContext context, ITypeSymbol attributeType) { - var allowConverterFallbackAttributeType = context.Compilation.GetTypeByMetadataName(Constants.Types.AllowConverterFallbackAttribute); - var allowConverterFallbackAttribute = attributeType.GetAttributes().FirstOrDefault(attr => SymbolEqualityComparer.Default.Equals(attr.AttributeClass, allowConverterFallbackAttributeType)); - return allowConverterFallbackAttribute.ConstructorArguments.FirstOrDefault().Value; + var converterFallbackBehaviorAttributeType = context.Compilation.GetTypeByMetadataName(Constants.Types.ConverterFallbackBehaviorAttribute); + var converterFallbackBehaviorAttribute = attributeType.GetAttributes().FirstOrDefault(attr => SymbolEqualityComparer.Default.Equals(attr.AttributeClass, converterFallbackBehaviorAttributeType)); + return converterFallbackBehaviorAttribute.ConstructorArguments.FirstOrDefault().Value; } private static List GetSupportedTypes(SymbolAnalysisContext context, List inputConverterAttributes) @@ -96,16 +98,16 @@ private static List GetSupportedTypes(SymbolAnalysisContext context, Lis var converterAttributes = converter.GetAttributes(); - var supportedConverterTypeAttributeType = context.Compilation.GetTypeByMetadataName(Constants.Types.SupportedConverterTypeAttribute); - var converterHasSupportedTypeAttribute = converterAttributes.Any(a => SymbolEqualityComparer.Default.Equals(a.AttributeClass, supportedConverterTypeAttributeType)); + var supportedTargetTypeAttributeType = context.Compilation.GetTypeByMetadataName(Constants.Types.SupportedTargetTypeAttribute); + var converterHasSupportedTypeAttribute = converterAttributes.Any(a => SymbolEqualityComparer.Default.Equals(a.AttributeClass, supportedTargetTypeAttributeType)); if (!converterHasSupportedTypeAttribute) { - // If a converter does not have the `SupportedConverterTypeAttribute`, we don't need to check for supported types + // If a converter does not have the `SupportedTargetTypeAttribute`, we don't need to check for supported types continue; } supportedTypes.AddRange(converterAttributes - .Where(a => SymbolEqualityComparer.Default.Equals(a.AttributeClass, supportedConverterTypeAttributeType)) + .Where(a => SymbolEqualityComparer.Default.Equals(a.AttributeClass, supportedTargetTypeAttributeType)) .SelectMany(a => a.ConstructorArguments.Select(arg => arg.Value)) .ToList()); } diff --git a/sdk/Sdk.Analyzers/Constants.cs b/sdk/Sdk.Analyzers/Constants.cs index 2f084c8e9..44a44174d 100644 --- a/sdk/Sdk.Analyzers/Constants.cs +++ b/sdk/Sdk.Analyzers/Constants.cs @@ -13,8 +13,8 @@ internal static class Types public const string InputBindingAttribute = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.InputBindingAttribute"; public const string TriggerBindingAttribute = "Microsoft.Azure.Functions.Worker.Extensions.Abstractions.TriggerBindingAttribute"; public const string InputConverterAttribute = "Microsoft.Azure.Functions.Worker.Converters.InputConverterAttribute"; - public const string AllowConverterFallbackAttribute = "Microsoft.Azure.Functions.Worker.Converters.AllowConverterFallbackAttribute"; - public const string SupportedConverterTypeAttribute = "Microsoft.Azure.Functions.Worker.Converters.SupportedConverterTypeAttribute"; + public const string ConverterFallbackBehaviorAttribute = "Microsoft.Azure.Functions.Worker.Converters.ConverterFallbackBehaviorAttribute"; + public const string SupportedTargetTypeAttribute = "Microsoft.Azure.Functions.Worker.Converters.SupportedTargetTypeAttribute"; // System types internal const string TaskType = "System.Threading.Tasks.Task"; From ed96e69707748d24f93c51a9d0349eb7a2dfe59b Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Thu, 13 Jul 2023 11:50:37 -0700 Subject: [PATCH 10/47] Temporarily disable BindingTypeCodeRefactoringProvider tests --- ...BindingTypeCodeRefactoringProviderTests.cs | 142 +++++++++--------- 1 file changed, 72 insertions(+), 70 deletions(-) diff --git a/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs b/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs index ff97c9cba..2394001b7 100644 --- a/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs +++ b/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs @@ -9,6 +9,8 @@ namespace Sdk.Analyzers.Tests { + // Disabling tests as they depend on Tables and ServiceBus extension releases with new deferred binding model changes + // Issue #1746 created to re-enable tests once new releases are available public class BindingTypeCodeRefactoringProviderTests : CodeRefactoringTestFixture { protected override string LanguageName => LanguageNames.CSharp; @@ -32,84 +34,84 @@ protected override CodeRefactoringProvider CreateProvider() ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.ServiceBus, Version=5.10.0.0").Location), }; - [Theory] - [InlineData("TableClient", 0)] - [InlineData("TableEntity", 1)] - public void TableInput_SuggestsCodeRefactor(string supportedType, int index) - { - string testCode = @" - using System; - using Azure.Data.Tables; - using Microsoft.Azure.Functions.Worker; + // [Theory] + // [InlineData("TableClient", 0)] + // [InlineData("TableEntity", 1)] + // public void TableInput_SuggestsCodeRefactor(string supportedType, int index) + // { + // string testCode = @" + // using System; + // using Azure.Data.Tables; + // using Microsoft.Azure.Functions.Worker; - namespace FunctionApp - { - public static class SomeFunction - { - [Function(nameof(SomeFunction))] - public static void Run([TableInput(""input"")] [|string message|]) - { - } - } - }"; + // namespace FunctionApp + // { + // public static class SomeFunction + // { + // [Function(nameof(SomeFunction))] + // public static void Run([TableInput(""input"")] [|string message|]) + // { + // } + // } + // }"; - string expectedCode = $@" - using System; - using Azure.Data.Tables; - using Microsoft.Azure.Functions.Worker; + // string expectedCode = $@" + // using System; + // using Azure.Data.Tables; + // using Microsoft.Azure.Functions.Worker; - namespace FunctionApp - {{ - public static class SomeFunction - {{ - [Function(nameof(SomeFunction))] - public static void Run([TableInput(""input"")] {supportedType} message) - {{ - }} - }} - }}"; + // namespace FunctionApp + // {{ + // public static class SomeFunction + // {{ + // [Function(nameof(SomeFunction))] + // public static void Run([TableInput(""input"")] {supportedType} message) + // {{ + // }} + // }} + // }}"; - TestCodeRefactoring(testCode, expectedCode, index); - } + // TestCodeRefactoring(testCode, expectedCode, index); + // } - [Theory] - [InlineData("ServiceBusReceivedMessage", 0)] - [InlineData("ServiceBusReceivedMessage[]", 1)] - public void ServiceBusTrigger_SuggestsCodeRefactor(string supportedType, int index) - { - string testCode = @" - using System; - using Azure.Messaging.ServiceBus; - using Microsoft.Azure.Functions.Worker; + // [Theory] + // [InlineData("ServiceBusReceivedMessage", 0)] + // [InlineData("ServiceBusReceivedMessage[]", 1)] + // public void ServiceBusTrigger_SuggestsCodeRefactor(string supportedType, int index) + // { + // string testCode = @" + // using System; + // using Azure.Messaging.ServiceBus; + // using Microsoft.Azure.Functions.Worker; - namespace FunctionApp - { - public static class SomeFunction - { - [Function(nameof(SomeFunction))] - public static void Run([ServiceBusTrigger(""input"")] [|string message|]) - { - } - } - }"; + // namespace FunctionApp + // { + // public static class SomeFunction + // { + // [Function(nameof(SomeFunction))] + // public static void Run([ServiceBusTrigger(""input"")] [|string message|]) + // { + // } + // } + // }"; - string expectedCode = $@" - using System; - using Azure.Messaging.ServiceBus; - using Microsoft.Azure.Functions.Worker; + // string expectedCode = $@" + // using System; + // using Azure.Messaging.ServiceBus; + // using Microsoft.Azure.Functions.Worker; - namespace FunctionApp - {{ - public static class SomeFunction - {{ - [Function(nameof(SomeFunction))] - public static void Run([ServiceBusTrigger(""input"")] {supportedType} message) - {{ - }} - }} - }}"; + // namespace FunctionApp + // {{ + // public static class SomeFunction + // {{ + // [Function(nameof(SomeFunction))] + // public static void Run([ServiceBusTrigger(""input"")] {supportedType} message) + // {{ + // }} + // }} + // }}"; - TestCodeRefactoring(testCode, expectedCode, index); - } + // TestCodeRefactoring(testCode, expectedCode, index); + // } } } From a17eefbd802caf7a59dc0591a24d6cf2033f0a9b Mon Sep 17 00:00:00 2001 From: Brett Samblanet Date: Tue, 18 Jul 2023 14:02:07 -0700 Subject: [PATCH 11/47] App Insights: improving property mappings; adding role details (#1755) --- .../DotNetWorker.ApplicationInsights.csproj | 2 +- .../FunctionsApplicationInsightsExtensions.cs | 22 +++++ .../FunctionsRoleInstanceProvider.cs | 34 ++++++++ .../FunctionsTelemetryInitializer.cs | 42 ---------- ...tionsRoleEnvironmentTelmetryInitializer.cs | 84 +++++++++++++++++++ .../FunctionsTelemetryInitializer.cs | 77 +++++++++++++++++ ...ce.cs => FunctionActivitySourceFactory.cs} | 0 .../ApplicationInsightsConfigurationTests.cs | 1 + .../FunctionsTelemetryInitializerTests.cs | 54 +++++++++++- 9 files changed, 269 insertions(+), 47 deletions(-) create mode 100644 src/DotNetWorker.ApplicationInsights/FunctionsRoleInstanceProvider.cs delete mode 100644 src/DotNetWorker.ApplicationInsights/FunctionsTelemetryInitializer.cs create mode 100644 src/DotNetWorker.ApplicationInsights/Initializers/FunctionsRoleEnvironmentTelmetryInitializer.cs create mode 100644 src/DotNetWorker.ApplicationInsights/Initializers/FunctionsTelemetryInitializer.cs rename src/DotNetWorker.Core/Diagnostics/{FunctionActivitySource.cs => FunctionActivitySourceFactory.cs} (100%) diff --git a/src/DotNetWorker.ApplicationInsights/DotNetWorker.ApplicationInsights.csproj b/src/DotNetWorker.ApplicationInsights/DotNetWorker.ApplicationInsights.csproj index 1ff462669..5079e4aa0 100644 --- a/src/DotNetWorker.ApplicationInsights/DotNetWorker.ApplicationInsights.csproj +++ b/src/DotNetWorker.ApplicationInsights/DotNetWorker.ApplicationInsights.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/DotNetWorker.ApplicationInsights/FunctionsApplicationInsightsExtensions.cs b/src/DotNetWorker.ApplicationInsights/FunctionsApplicationInsightsExtensions.cs index 7342bc612..a80496cb3 100644 --- a/src/DotNetWorker.ApplicationInsights/FunctionsApplicationInsightsExtensions.cs +++ b/src/DotNetWorker.ApplicationInsights/FunctionsApplicationInsightsExtensions.cs @@ -2,11 +2,14 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights.Extensibility.PerfCounterCollector.QuickPulse; using Microsoft.Azure.Functions.Worker.ApplicationInsights; +using Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; @@ -23,7 +26,26 @@ public static IServiceCollection ConfigureFunctionsApplicationInsights(this ISer throw new ArgumentNullException(nameof(services)); } + services.AddSingleton(); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + services.AddSingleton(provider => + { + // To match parity with the Host, we need to update the QuickPulseTelemetryModule.ServerId. We don't want to reference the + // top-level WorkerService or AspNetCore packages, so we cannot use ConfigureTelemetryModules(). + // + // Nesting this setup inside this ITelemetryInitializer factory as it guarantees it will be run before + // any ITelemetryModules are initialized. + var modules = provider.GetServices(); + var quickPulseModule = modules.OfType().SingleOrDefault(); + if (quickPulseModule is not null) + { + var roleInstanceProvider = provider.GetRequiredService(); + quickPulseModule.ServerId = roleInstanceProvider.GetRoleInstanceName(); + } + + return ActivatorUtilities.CreateInstance(provider); + }); services.TryAddEnumerable(ServiceDescriptor.Singleton()); services.AddOptions() .Validate( diff --git a/src/DotNetWorker.ApplicationInsights/FunctionsRoleInstanceProvider.cs b/src/DotNetWorker.ApplicationInsights/FunctionsRoleInstanceProvider.cs new file mode 100644 index 000000000..f3be1f0a1 --- /dev/null +++ b/src/DotNetWorker.ApplicationInsights/FunctionsRoleInstanceProvider.cs @@ -0,0 +1,34 @@ +using System; + +namespace Microsoft.Azure.Functions.Worker.ApplicationInsights +{ + internal class FunctionsRoleInstanceProvider + { + internal const string ComputerNameKey = "COMPUTERNAME"; + internal const string WebSiteInstanceIdKey = "WEBSITE_INSTANCE_ID"; + internal const string ContainerNameKey = "CONTAINER_NAME"; + + private string? _roleInstanceName; + + public string GetRoleInstanceName() + { + _roleInstanceName ??= GetRoleInstance(); + return _roleInstanceName; + } + + private static string GetRoleInstance() + { + string instanceName = Environment.GetEnvironmentVariable(WebSiteInstanceIdKey); + if (string.IsNullOrEmpty(instanceName)) + { + instanceName = Environment.GetEnvironmentVariable(ComputerNameKey); + if (string.IsNullOrEmpty(instanceName)) + { + instanceName = Environment.GetEnvironmentVariable(ContainerNameKey); + } + } + + return instanceName; + } + } +} diff --git a/src/DotNetWorker.ApplicationInsights/FunctionsTelemetryInitializer.cs b/src/DotNetWorker.ApplicationInsights/FunctionsTelemetryInitializer.cs deleted file mode 100644 index 06e1a6a38..000000000 --- a/src/DotNetWorker.ApplicationInsights/FunctionsTelemetryInitializer.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Reflection; -using Microsoft.ApplicationInsights.Channel; -using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.ApplicationInsights.Extensibility.Implementation; - -namespace Microsoft.Azure.Functions.Worker.ApplicationInsights -{ - internal class FunctionsTelemetryInitializer : ITelemetryInitializer - { - private readonly string? _sdkVersion; - - internal FunctionsTelemetryInitializer(string? sdkVersion) - { - _sdkVersion = sdkVersion; - } - - public FunctionsTelemetryInitializer() : - this(GetSdkVersion()) - { - } - - private static string? GetSdkVersion() - { - string? version = typeof(FunctionsTelemetryInitializer).Assembly.GetCustomAttribute()?.InformationalVersion; - - return version == null ? null : $"azurefunctions-netiso: {version}"; - } - - public void Initialize(ITelemetry telemetry) - { - if (telemetry is null || _sdkVersion is null) - { - return; - } - - telemetry.Context.GetInternalContext().SdkVersion = _sdkVersion; - } - } -} diff --git a/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsRoleEnvironmentTelmetryInitializer.cs b/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsRoleEnvironmentTelmetryInitializer.cs new file mode 100644 index 000000000..91d7f02f2 --- /dev/null +++ b/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsRoleEnvironmentTelmetryInitializer.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights.Extensibility.Implementation; + +namespace Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers +{ + // This class was taken largely from https://raw.githubusercontent.com/Microsoft/ApplicationInsights-dotnet-server/91016d62f3181e10d4cf589ef8fd64dadb6b54a2/Src/WindowsServer/WindowsServer.Shared/AzureWebAppRoleEnvironmentTelemetryInitializer.cs, + // but refactored so that it did not use WEBSITE_HOSTNAME, which is determined to be unreliable for functions during slot swaps. + + /// + /// A telemetry initializer that will gather Azure Web App Role Environment context information. + /// + internal class FunctionsRoleEnvironmentTelemetryInitializer : ITelemetryInitializer + { + internal const string AzureWebsiteName = "WEBSITE_SITE_NAME"; + internal const string AzureWebsiteSlotName = "WEBSITE_SLOT_NAME"; + internal const string AzureWebsiteCloudRoleName = "WEBSITE_CLOUD_ROLENAME"; + private const string DefaultProductionSlotName = "production"; + private const string WebAppSuffix = ".azurewebsites.net"; + + private readonly ConcurrentDictionary _siteNodeNames = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Initializes device context. + /// + /// The telemetry to initialize. + public void Initialize(ITelemetry telemetry) + { + if (telemetry == null) + { + return; + } + + var siteSlotName = new Lazy(() => + { + // We cannot cache these values as the environment variables can change on the fly. + return GetAzureWebsiteUniqueSlotName(); + }); + + var websiteCloudRoleName = Environment.GetEnvironmentVariable(AzureWebsiteCloudRoleName); + + if (!string.IsNullOrEmpty(websiteCloudRoleName)) + { + telemetry.Context.Cloud.RoleName = websiteCloudRoleName; + } + else + { + telemetry.Context.Cloud.RoleName = siteSlotName.Value; + } + + var internalContext = telemetry.Context.GetInternalContext(); + if (!string.IsNullOrEmpty(siteSlotName.Value)) + { + internalContext.NodeName = _siteNodeNames.GetOrAdd(siteSlotName.Value!, p => + { + // maintain previous behavior of node having the full url + return p += WebAppSuffix; + }); + } + } + + /// + /// Gets a value that uniquely identifies the site and slot. + /// + private static string? GetAzureWebsiteUniqueSlotName() + { + var name = Environment.GetEnvironmentVariable(AzureWebsiteName); + var slotName = Environment.GetEnvironmentVariable(AzureWebsiteSlotName); + + if (!string.IsNullOrEmpty(slotName) && + !string.Equals(slotName, DefaultProductionSlotName, StringComparison.OrdinalIgnoreCase)) + { + name += $"-{slotName}"; + } + + return name?.ToLowerInvariant(); + } + } +} diff --git a/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsTelemetryInitializer.cs b/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsTelemetryInitializer.cs new file mode 100644 index 000000000..2bdc35414 --- /dev/null +++ b/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsTelemetryInitializer.cs @@ -0,0 +1,77 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Reflection; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights.Extensibility.Implementation; + +namespace Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers +{ + internal class FunctionsTelemetryInitializer : ITelemetryInitializer + { + private readonly string _roleInstanceName; + private readonly string? _sdkVersion; + private static readonly IDictionary _mappings = new Dictionary() + { + { "faas.execution", "InvocationId" }, // used by worker ActivitySource + { "AzureFunctions_InvocationId", "InvocationId" } // used by log scope + }; + + internal FunctionsTelemetryInitializer(FunctionsRoleInstanceProvider roleInstanceProvider, string? sdkVersion) + { + _roleInstanceName = roleInstanceProvider.GetRoleInstanceName(); + _sdkVersion = sdkVersion; + } + + public FunctionsTelemetryInitializer(FunctionsRoleInstanceProvider roleInstanceProvider) : + this(roleInstanceProvider, GetSdkVersion()) + { + } + + private static string? GetSdkVersion() + { + var version = typeof(FunctionsTelemetryInitializer).Assembly.GetCustomAttribute()?.InformationalVersion; + + return version == null ? null : $"azurefunctions-netiso: {version}"; + } + + public void Initialize(ITelemetry telemetry) + { + if (telemetry is null) + { + return; + } + + if (_sdkVersion is not null) + { + telemetry.Context.GetInternalContext().SdkVersion = _sdkVersion; + } + + if (_roleInstanceName is not null) + { + telemetry.Context.Cloud.RoleInstance = _roleInstanceName; + } + + if (telemetry is ISupportProperties supportProperties) + { + CopyWellKnownProperties(supportProperties); + } + } + + // For parity with how the Functions host writes to App Insights, make sure we translate some + // well-known worker keys into the ones used by the host. + internal static void CopyWellKnownProperties(ISupportProperties supportProperties) + { + foreach (var mapping in _mappings) + { + if (supportProperties.Properties.TryGetValue(mapping.Key, out string propValue)) + { + supportProperties.Properties[mapping.Value] = propValue; + } + } + } + } +} diff --git a/src/DotNetWorker.Core/Diagnostics/FunctionActivitySource.cs b/src/DotNetWorker.Core/Diagnostics/FunctionActivitySourceFactory.cs similarity index 100% rename from src/DotNetWorker.Core/Diagnostics/FunctionActivitySource.cs rename to src/DotNetWorker.Core/Diagnostics/FunctionActivitySourceFactory.cs diff --git a/test/DotNetWorkerTests/ApplicationInsights/ApplicationInsightsConfigurationTests.cs b/test/DotNetWorkerTests/ApplicationInsights/ApplicationInsightsConfigurationTests.cs index 424c38d90..46f6f11c6 100644 --- a/test/DotNetWorkerTests/ApplicationInsights/ApplicationInsightsConfigurationTests.cs +++ b/test/DotNetWorkerTests/ApplicationInsights/ApplicationInsightsConfigurationTests.cs @@ -3,6 +3,7 @@ using System.Linq; using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Azure.Functions.Worker.ApplicationInsights; +using Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; diff --git a/test/DotNetWorkerTests/ApplicationInsights/FunctionsTelemetryInitializerTests.cs b/test/DotNetWorkerTests/ApplicationInsights/FunctionsTelemetryInitializerTests.cs index d5f16d788..07b3c798c 100644 --- a/test/DotNetWorkerTests/ApplicationInsights/FunctionsTelemetryInitializerTests.cs +++ b/test/DotNetWorkerTests/ApplicationInsights/FunctionsTelemetryInitializerTests.cs @@ -1,20 +1,66 @@ -using Microsoft.ApplicationInsights.DataContracts; +using System; +using Microsoft.ApplicationInsights.DataContracts; using Microsoft.ApplicationInsights.Extensibility.Implementation; using Microsoft.Azure.Functions.Worker.ApplicationInsights; +using Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers; using Xunit; namespace Microsoft.Azure.Functions.Worker.Tests.ApplicationInsights; -public class FunctionsTelemetryInitializerTests +public class FunctionsTelemetryInitializerTests : IDisposable { + public FunctionsTelemetryInitializerTests() + { + // make sure these are clear before each test + SetEnvironmentVariables(null, null, null); + } + [Fact] public void Initialize_SetsContextProperties() { - var initializer = new FunctionsTelemetryInitializer("testversion"); + var initializer = new FunctionsTelemetryInitializer(new FunctionsRoleInstanceProvider(), "testversion"); var telemetry = new TraceTelemetry(); initializer.Initialize(telemetry); Assert.Equal("testversion", telemetry.Context.GetInternalContext().SdkVersion); } -} + [Fact] + public void RoleInstanceProvider_UsesWebsiteInstanceId() + { + SetEnvironmentVariables("instanceId", "computerName", "containerName"); + + var provider = new FunctionsRoleInstanceProvider(); + Assert.Equal("instanceId", provider.GetRoleInstanceName()); + } + + [Fact] + public void RoleInstanceProvider_UsesComputerName() + { + SetEnvironmentVariables(null, "computerName", "containerName"); + + var provider = new FunctionsRoleInstanceProvider(); + Assert.Equal("computerName", provider.GetRoleInstanceName()); + } + + [Fact] + public void RoleInstanceProvider_UsesContainerName() + { + SetEnvironmentVariables(null, null, "containerName"); + + var provider = new FunctionsRoleInstanceProvider(); + Assert.Equal("containerName", provider.GetRoleInstanceName()); + } + + private static void SetEnvironmentVariables(string instanceId, string computerName, string containerName) + { + Environment.SetEnvironmentVariable(FunctionsRoleInstanceProvider.WebSiteInstanceIdKey, instanceId); + Environment.SetEnvironmentVariable(FunctionsRoleInstanceProvider.ComputerNameKey, computerName); + Environment.SetEnvironmentVariable(FunctionsRoleInstanceProvider.ContainerNameKey, containerName); + } + + public void Dispose() + { + SetEnvironmentVariables(null, null, null); + } +} From 697cb8e2bb9de3b65a088e1d5eabb487302ec106 Mon Sep 17 00:00:00 2001 From: Surgupta Date: Wed, 18 Jan 2023 10:47:12 -0800 Subject: [PATCH 12/47] Assign cardinality correctly for Blob collection scenarios (#1271) * IsBatched for blobInputAttribute --- .../FunctionMetadataGeneratorTests.cs | 12 ------------ .../IntegratedTriggersAndBindingsTests.cs | 2 +- .../StorageBindingTests.cs | 6 +++--- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 8fc92efb7..5778409cd 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -750,18 +750,6 @@ public object BlobToBlobs( } } - private class SDKTypeBindings - { - [Function("BlobToBlobFunction")] - [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] - public object BlobToBlob( - [BlobTrigger("container2/%file%")] string blob, - [BlobInput("container2/%file%")] string blobinput) - { - throw new NotImplementedException(); - } - } - private class ExternalType_Return { public const string FunctionName = "BasicHttpWithExternalTypeReturn"; diff --git a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs index 445f93643..bf2b86883 100644 --- a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs +++ b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs @@ -213,7 +213,7 @@ public Task> GetFunctionMetadataAsync(string d { var metadataList = new List(); var Function0RawBindings = new List(); - Function0RawBindings.Add(@"{""name"":""req"",""type"":""HttpTrigger"",""direction"":""In"",""authLevel"":""Anonymous"",""methods"":[""get"",""post""]}"); + Function0RawBindings.Add(@"{""name"":""req"",""type"":""HttpTrigger"",""direction"":""In"",""methods"":[""get"",""post""]}"); Function0RawBindings.Add(@"{""name"":""myBlob"",""type"":""Blob"",""direction"":""In"",""blobPath"":""test-samples/sample1.txt"",""connection"":""AzureWebJobsStorage"",""cardinality"":""One"",""dataType"":""String""}"); Function0RawBindings.Add(@"{""name"":""Book"",""type"":""Queue"",""direction"":""Out"",""queueName"":""functionstesting2"",""connection"":""AzureWebJobsStorage""}"); Function0RawBindings.Add(@"{""name"":""HttpResponse"",""type"":""http"",""direction"":""Out""}"); diff --git a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs index 2195ce7f8..c0d0ff211 100644 --- a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs +++ b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs @@ -219,13 +219,13 @@ public Task> GetFunctionMetadataAsync(string d metadataList.Add(Function1); var Function2RawBindings = new List(); Function2RawBindings.Add(@"{""name"":""$return"",""type"":""Queue"",""direction"":""Out"",""queueName"":""queue2""}"); - Function2RawBindings.Add(@"{""name"":""blobs"",""type"":""Blob"",""direction"":""In"",""blobPath"":""container2"",""cardinality"":""Many"",""dataType"":""String""}"); - + Function2RawBindings.Add(@"{""name"":""blobs"",""type"":""Blob"",""direction"":""In"",""blobPath"":""container2"",""cardinality"":""Many""}"); + var Function2 = new DefaultFunctionMetadata { Language = "dotnet-isolated", Name = "BlobsToQueueFunction", - EntryPoint = "FunctionApp.QueueTriggerAndOutput.BlobsToQueue", + EntryPoint = "TestProject.QueueTriggerAndOutput.BlobsToQueue", RawBindings = Function2RawBindings, ScriptFile = "TestProject.dll" }; From 48f910baf3235063ad2411e7b4c139243f105d75 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 8 Feb 2023 15:50:43 -0800 Subject: [PATCH 13/47] Implement blob converter for SDK-type binding support (#1108) Co-authored-by: Surgupta --- .../release_notes.md | 3 +- .../src/BlobInputAttribute.cs | 1 + .../src/BlobStorageConverter.cs | 236 ++++++++++++++++++ .../src/BlobTriggerAttribute.cs | 1 + .../src/StorageExtensionStartup.cs | 5 + .../FunctionMetadataGeneratorTests.cs | 78 +++++- .../IntegratedTriggersAndBindingsTests.cs | 37 +++ test/SdkE2ETests/Contents/functions.metadata | 10 +- test/SdkE2ETests/PublishTests.cs | 2 +- 9 files changed, 365 insertions(+), 8 deletions(-) create mode 100644 extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs diff --git a/extensions/Worker.Extensions.Storage.Blobs/release_notes.md b/extensions/Worker.Extensions.Storage.Blobs/release_notes.md index a6b1d13a9..f13105d3c 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/release_notes.md +++ b/extensions/Worker.Extensions.Storage.Blobs/release_notes.md @@ -6,4 +6,5 @@ ### Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs -- +- Add support for SDK-type bindings via deferred binding feature #1108 +- Assign cardinality correctly for Blob collection scenarios #1271 diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs index a2856cad2..78ab31abc 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Functions.Worker { + [SupportsDeferredBinding] public sealed class BlobInputAttribute : InputBindingAttribute { private readonly string _blobPath; diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs new file mode 100644 index 000000000..3e0a6c96f --- /dev/null +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -0,0 +1,236 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.Functions.Worker +{ + /// + /// Converter to bind Blob Storage type parameters. + /// + internal class BlobStorageConverter : IInputConverter + { + private readonly IOptions _workerOptions; + private readonly IOptionsSnapshot _blobOptions; + + private readonly ILogger _logger; + + public BlobStorageConverter(IOptions workerOptions, IOptionsSnapshot blobOptions, ILogger logger) + { + _workerOptions = workerOptions ?? throw new ArgumentNullException(nameof(workerOptions)); + _blobOptions = blobOptions ?? throw new ArgumentNullException(nameof(blobOptions)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask ConvertAsync(ConverterContext context) + { + return context?.Source switch + { + ModelBindingData binding => await ConvertFromBindingDataAsync(context, binding), + CollectionModelBindingData binding => await ConvertFromCollectionBindingDataAsync(context, binding), + _ => ConversionResult.Unhandled(), + }; + } + + private async ValueTask ConvertFromBindingDataAsync(ConverterContext context, ModelBindingData modelBindingData) + { + if (!IsBlobExtension(modelBindingData)) + { + return ConversionResult.Unhandled(); + } + + try + { + Dictionary content = GetBindingDataContent(modelBindingData); + var result = await ConvertModelBindingDataAsync(content, context.TargetType, modelBindingData); + + if (result is not null) + { + return ConversionResult.Success(result); + } + } + catch (Exception ex) + { + return ConversionResult.Failed(ex); + } + + return ConversionResult.Unhandled(); + } + + private async ValueTask ConvertFromCollectionBindingDataAsync(ConverterContext context, CollectionModelBindingData collectionModelBindingData) + { + var blobCollection = new List(collectionModelBindingData.ModelBindingDataArray.Length); + Type elementType = context.TargetType.IsArray ? context.TargetType.GetElementType() : context.TargetType.GenericTypeArguments[0]; + + try + { + foreach (ModelBindingData modelBindingData in collectionModelBindingData.ModelBindingDataArray) + { + if (!IsBlobExtension(modelBindingData)) + { + return ConversionResult.Unhandled(); + } + + Dictionary content = GetBindingDataContent(modelBindingData); + var element = await ConvertModelBindingDataAsync(content, elementType, modelBindingData); + + if (element is not null) + { + blobCollection.Add(element); + } + } + + var methodName = context.TargetType.IsArray ? nameof(CloneToArray) : nameof(CloneToList); + var result = ToTargetTypeCollection(blobCollection, methodName, elementType); + + return ConversionResult.Success(result); + } + catch (Exception ex) + { + return ConversionResult.Failed(ex); + } + } + + private bool IsBlobExtension(ModelBindingData bindingData) + { + if (bindingData?.Source is not Constants.BlobExtensionName) + { + _logger.LogTrace("Source '{source}' is not supported by {converter}", bindingData?.Source, nameof(BlobStorageConverter)); + return false; + } + + return true; + } + + private Dictionary GetBindingDataContent(ModelBindingData bindingData) + { + return bindingData?.ContentType switch + { + Constants.JsonContentType => new Dictionary(bindingData?.Content?.ToObjectFromJson>(), StringComparer.OrdinalIgnoreCase), + _ => throw new NotSupportedException($"Unexpected content-type. Currently only {Constants.JsonContentType} is supported.") + }; + } + + private async Task ConvertModelBindingDataAsync(IDictionary content, Type targetType, ModelBindingData bindingData) + { + content.TryGetValue(Constants.Connection, out var connectionName); + content.TryGetValue(Constants.ContainerName, out var containerName); + content.TryGetValue(Constants.BlobName, out var blobName); + + if (string.IsNullOrEmpty(connectionName) || string.IsNullOrEmpty(containerName)) + { + throw new ArgumentNullException("'Connection' and 'ContainerName' cannot be null or empty"); + } + + return await ToTargetTypeAsync(targetType, connectionName, containerName, blobName); + } + + private async Task ToTargetTypeAsync(Type targetType, string connectionName, string containerName, string blobName) => targetType switch + { + Type _ when targetType == typeof(String) => await GetBlobStringAsync(connectionName, containerName, blobName), + Type _ when targetType == typeof(Stream) => await GetBlobStreamAsync(connectionName, containerName, blobName), + Type _ when targetType == typeof(Byte[]) => await GetBlobBinaryDataAsync(connectionName, containerName, blobName), + Type _ when targetType == typeof(BlobBaseClient) => CreateBlobClient(connectionName, containerName, blobName), + Type _ when targetType == typeof(BlobClient) => CreateBlobClient(connectionName, containerName, blobName), + Type _ when targetType == typeof(BlockBlobClient) => CreateBlobClient(connectionName, containerName, blobName), + Type _ when targetType == typeof(PageBlobClient) => CreateBlobClient(connectionName, containerName, blobName), + Type _ when targetType == typeof(AppendBlobClient) => CreateBlobClient(connectionName, containerName, blobName), + Type _ when targetType == typeof(BlobContainerClient) => CreateBlobContainerClient(connectionName, containerName), + _ => await DeserializeToTargetObjectAsync(targetType, connectionName, containerName, blobName) + }; + + private async Task DeserializeToTargetObjectAsync(Type targetType, string connectionName, string containerName, string blobName) + { + var content = await GetBlobStreamAsync(connectionName, containerName, blobName); + return _workerOptions?.Value?.Serializer?.Deserialize(content, targetType, CancellationToken.None); + } + + private object? ToTargetTypeCollection(IEnumerable blobCollection, string methodName, Type type) + { + blobCollection = blobCollection.Select(b => Convert.ChangeType(b, type)); + MethodInfo method = typeof(BlobStorageConverter).GetMethod(methodName, BindingFlags.Static | BindingFlags.NonPublic); + MethodInfo genericMethod = method.MakeGenericMethod(type); + + return genericMethod.Invoke(null, new[] { blobCollection.ToList() }); + } + + private static T[] CloneToArray(IList source) + { + return source.Cast().ToArray(); + } + + private static IEnumerable CloneToList(IList source) + { + return source.Cast(); + } + + private async Task GetBlobStringAsync(string connectionName, string containerName, string blobName) + { + var client = CreateBlobClient(connectionName, containerName, blobName); + return await GetBlobContentStringAsync(client); + } + + private async Task GetBlobContentStringAsync(BlobClient client) + { + var download = await client.DownloadContentAsync(); + return download.Value.Content.ToString(); + } + + private async Task GetBlobBinaryDataAsync(string connectionName, string containerName, string blobName) + { + using MemoryStream stream = new(); + var client = CreateBlobClient(connectionName, containerName, blobName); + await client.DownloadToAsync(stream); + return stream.ToArray(); + } + + private async Task GetBlobStreamAsync(string connectionName, string containerName, string blobName) + { + var client = CreateBlobClient(connectionName, containerName, blobName); + var download = await client.DownloadStreamingAsync(); + return download.Value.Content; + } + + private BlobContainerClient CreateBlobContainerClient(string connectionName, string containerName) + { + var blobStorageOptions = _blobOptions.Get(connectionName); + BlobServiceClient blobServiceClient = blobStorageOptions.CreateClient(); + BlobContainerClient container = blobServiceClient.GetBlobContainerClient(containerName); + return container; + } + + private T CreateBlobClient(string connectionName, string containerName, string blobName) where T : BlobBaseClient + { + if (string.IsNullOrEmpty(blobName)) + { + throw new ArgumentNullException(nameof(blobName)); + } + + BlobContainerClient container = CreateBlobContainerClient(connectionName, containerName); + + Type targetType = typeof(T); + BlobBaseClient blobClient = targetType switch + { + Type _ when targetType == typeof(BlobClient) => container.GetBlobClient(blobName), + Type _ when targetType == typeof(BlockBlobClient) => container.GetBlockBlobClient(blobName), + Type _ when targetType == typeof(PageBlobClient) => container.GetPageBlobClient(blobName), + Type _ when targetType == typeof(AppendBlobClient) => container.GetAppendBlobClient(blobName), + _ => container.GetBlobBaseClient(blobName) + }; + + return (T)blobClient; + } + } +} \ No newline at end of file diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs index 3a546c6be..59ac9fbc8 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Functions.Worker { + [SupportsDeferredBinding] public sealed class BlobTriggerAttribute : TriggerBindingAttribute { private readonly string _blobPath; diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/StorageExtensionStartup.cs b/extensions/Worker.Extensions.Storage.Blobs/src/StorageExtensionStartup.cs index 9f8b10bb1..a7bee100d 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/StorageExtensionStartup.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/StorageExtensionStartup.cs @@ -24,6 +24,11 @@ public override void Configure(IFunctionsWorkerApplicationBuilder applicationBui applicationBuilder.Services.AddAzureClientsCore(); // Adds AzureComponentFactory applicationBuilder.Services.AddOptions(); applicationBuilder.Services.AddSingleton, BlobStorageBindingOptionsSetup>(); + + applicationBuilder.Services.Configure((workerOption) => + { + workerOption.InputConverters.RegisterAt(0); + }); } } } diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 5778409cd..f1b1effa2 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -210,7 +210,7 @@ void ValidateBlobInput(ExpandoObject b) { "Direction", "In" }, { "blobPath", "container2" }, { "Cardinality", "Many" }, - { "Properties", new Dictionary() } + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } }); } @@ -227,7 +227,7 @@ void ValidateBlobTrigger(ExpandoObject b) { "Direction", "In" }, { "DataType", "String"}, { "path", "container2/%file%" }, - { "Properties", new Dictionary() } + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } }); } @@ -244,6 +244,68 @@ void ValidateQueueOutput(ExpandoObject b) } } + [Fact] + public void StorageFunction_SDKTypeBindings() + { + var generator = new FunctionMetadataGenerator(); + var module = ModuleDefinition.ReadModule(_thisAssembly.Location); + var typeDef = TestUtility.GetTypeDefinition(typeof(SDKTypeBindings)); + var functions = generator.GenerateFunctionMetadata(typeDef); + var extensions = generator.Extensions; + + Assert.Equal(1, functions.Count()); + + var blobToBlob = functions.Single(p => p.Name == "BlobToBlobFunction"); + + ValidateFunction(blobToBlob, "BlobToBlobFunction", GetEntryPoint(nameof(SDKTypeBindings), nameof(SDKTypeBindings.BlobToBlob)), + b => ValidateBlobTrigger(b), + b => ValidateBlobInput(b), + b => ValidateBlobOutput(b)); + + AssertDictionary(extensions, new Dictionary + { + { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.0-beta.1" }, + }); + + void ValidateBlobTrigger(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "blob" }, + { "Type", "blobTrigger" }, + { "Direction", "In" }, + { "path", "container2/%file%" }, + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } + }); + } + + void ValidateBlobInput(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "blobinput" }, + { "Type", "blob" }, + { "Direction", "In" }, + { "blobPath", "container2/%file%" }, + { "Cardinality", "One" }, + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } + }); + } + + void ValidateBlobOutput(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "$return" }, + { "Type", "blob" }, + { "Direction", "Out" }, + { "blobPath", "container1/hello.txt" }, + { "Connection", "MyOtherConnection" }, + { "Properties", new Dictionary() } + }); + } + } + [Fact] public void TimerFunction() { @@ -750,6 +812,18 @@ public object BlobToBlobs( } } + private class SDKTypeBindings + { + [Function("BlobToBlobFunction")] + [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] + public object BlobToBlob( + [BlobTrigger("container2/%file%")] string blob, + [BlobInput("container2/%file%")] string blobinput) + { + throw new NotImplementedException(); + } + } + private class ExternalType_Return { public const string FunctionName = "BasicHttpWithExternalTypeReturn"; diff --git a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs index bf2b86883..b19c23437 100644 --- a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs +++ b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs @@ -526,6 +526,43 @@ await TestHelpers.RunTestAsync( expectedGeneratedFileName, expectedOutput); } + + [Fact (Skip = "Depends on source gen description cleanup, issue #1323")] + public async void MultipleOutputOnMethodFails() + { + var inputCode = @"using System; + using System.Net; + using System.Collections; + using System.Collections.Generic; + using Microsoft.Azure.Functions.Worker; + using Microsoft.Azure.Functions.Worker.Http; + using System.Linq; + using System.Threading.Tasks; + + namespace FunctionApp + { + public class EventHubsInput + { + [Function(""QueueToBlobFunction"")] + [BlobOutput(""container1/hello.txt"", Connection = ""MyOtherConnection"")] + [QueueOutput(""queue2"")] + public string QueueToBlob( + [QueueTrigger(""queueName"", Connection = ""MyConnection"")] string queuePayload) + { + throw new NotImplementedException(); + } + } + }"; + + string? expectedGeneratedFileName = null; + string? expectedOutput = null; + + await TestHelpers.RunTestAsync( + _referencedExtensionAssemblies, + inputCode, + expectedGeneratedFileName, + expectedOutput); + } } } } diff --git a/test/SdkE2ETests/Contents/functions.metadata b/test/SdkE2ETests/Contents/functions.metadata index b8445f79f..2f4d1173f 100644 --- a/test/SdkE2ETests/Contents/functions.metadata +++ b/test/SdkE2ETests/Contents/functions.metadata @@ -50,11 +50,12 @@ "name": "myBlob", "direction": "In", "type": "blob", - "dataType": "String", "blobPath": "test-samples/sample1.txt", "connection": "AzureWebJobsStorage", "cardinality": "One", - "properties": {} + "properties": { + "supportsDeferredBinding": "True" + } }, { "name": "Book", @@ -182,11 +183,12 @@ "name": "myBlob", "direction": "In", "type": "blob", - "dataType": "String", "blobPath": "test-samples/sample1.txt", "connection": "AzureWebJobsStorage", "cardinality": "One", - "properties": {} + "properties": { + "supportsDeferredBinding": "True" + } } ] } diff --git a/test/SdkE2ETests/PublishTests.cs b/test/SdkE2ETests/PublishTests.cs index 373e5fe0d..0e5d31297 100644 --- a/test/SdkE2ETests/PublishTests.cs +++ b/test/SdkE2ETests/PublishTests.cs @@ -64,7 +64,7 @@ private async Task RunPublishTest(string outputDir, string additionalParams = nu "Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.Startup, Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader, Version=1.0.0.0, Culture=neutral, PublicKeyToken=551316b6919f366c", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.dll"), new Extension("AzureStorageBlobs", - "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageBlobsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Blobs, Version=5.1.2.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", + "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageBlobsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Blobs, Version=5.1.0.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.dll"), new Extension("AzureStorageQueues", "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageQueuesWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Queues, Version=5.1.2.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", From 75fc6cb1e1384c4e75381c9246331212783192f5 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 22 Feb 2023 15:43:05 -0800 Subject: [PATCH 14/47] Add E2E tests for blob SDK type bindings (#1360) --- .../E2EApp/Blob/BlobInputBindingFunctions.cs | 134 ++++++++ .../E2EApps/E2EApp/Blob/BlobTestFunctions.cs | 57 ---- .../Blob/BlobTriggerBindingFunctions.cs | 90 ++++++ test/E2ETests/E2EApps/E2EApp/Blob/Book.cs | 11 + test/E2ETests/E2EApps/E2EApp/E2EApp.csproj | 5 +- test/E2ETests/E2ETests/Constants.cs | 18 +- .../{ => Cosmos}/CosmosDBEndToEndTests.cs | 2 +- .../E2ETests/Helpers/StorageHelpers.cs | 6 + .../E2ETests/Storage/BlobEndToEndTests.cs | 292 ++++++++++++++++++ .../QueueEndToEndTests.cs} | 73 +---- 10 files changed, 556 insertions(+), 132 deletions(-) create mode 100644 test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs delete mode 100644 test/E2ETests/E2EApps/E2EApp/Blob/BlobTestFunctions.cs create mode 100644 test/E2ETests/E2EApps/E2EApp/Blob/BlobTriggerBindingFunctions.cs create mode 100644 test/E2ETests/E2EApps/E2EApp/Blob/Book.cs rename test/E2ETests/E2ETests/{ => Cosmos}/CosmosDBEndToEndTests.cs (95%) create mode 100644 test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs rename test/E2ETests/E2ETests/{StorageEndToEndTests.cs => Storage/QueueEndToEndTests.cs} (76%) diff --git a/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs b/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs new file mode 100644 index 000000000..4e6798c90 --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs @@ -0,0 +1,134 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.Functions.Worker.E2EApp.Blob +{ + public class BlobInputBindingFunctions + { + private readonly ILogger _logger; + + public BlobInputBindingFunctions(ILogger logger) + { + _logger = logger; + } + + [Function(nameof(BlobInputClientTest))] + public async Task BlobInputClientTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] BlobClient client) + { + var downloadResult = await client.DownloadContentAsync(); + var response = req.CreateResponse(HttpStatusCode.OK); + await response.Body.WriteAsync(downloadResult.Value.Content); + return response; + } + + [Function(nameof(BlobInputContainerClientTest))] + public async Task BlobInputContainerClientTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] BlobContainerClient client) + { + var blobClient = client.GetBlobClient("testFile.txt"); + var downloadResult = await blobClient.DownloadContentAsync(); + var response = req.CreateResponse(HttpStatusCode.OK); + await response.Body.WriteAsync(downloadResult.Value.Content); + return response; + } + + [Function(nameof(BlobInputStreamTest))] + public async Task BlobInputStreamTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] Stream stream) + { + using var blobStreamReader = new StreamReader(stream); + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(blobStreamReader.ReadToEnd()); + return response; + } + + [Function(nameof(BlobInputByteTest))] + public async Task BlobInputByteTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] Byte[] data) + { + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(Encoding.Default.GetString(data)); + return response; + } + + [Function(nameof(BlobInputStringTest))] + public async Task BlobInputStringTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] string data) + { + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(data); + return response; + } + + [Function(nameof(BlobInputPocoTest))] + public async Task BlobInputPocoTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] Book data) + { + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(data.Name); + return response; + } + + [Function(nameof(BlobInputCollectionTest))] + public async Task BlobInputCollectionTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated", IsBatched = true)] IEnumerable blobs) + { + List blobList = new(); + + foreach (BlobClient blob in blobs) + { + _logger.LogInformation("Blob name: {blobName}, Container name: {containerName}", blob.Name, blob.BlobContainerName); + blobList.Add(blob.Name); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(blobList.ToString()); + return response; + } + + [Function(nameof(BlobInputStringArrayTest))] + public async Task BlobInputStringArrayTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated", IsBatched = true)] string[] blobContent) + { + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(blobContent.ToString()); + return response; + } + + [Function(nameof(BlobInputPocoArrayTest))] + public async Task BlobInputPocoArrayTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated", IsBatched = true)] Book[] books) + { + List bookNames = new(); + + foreach (var item in books) + { + bookNames.Add(item.Name); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(bookNames.ToString()); + return response; + } + } +} \ No newline at end of file diff --git a/test/E2ETests/E2EApps/E2EApp/Blob/BlobTestFunctions.cs b/test/E2ETests/E2EApps/E2EApp/Blob/BlobTestFunctions.cs deleted file mode 100644 index b375e044a..000000000 --- a/test/E2ETests/E2EApps/E2EApp/Blob/BlobTestFunctions.cs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Text.Json.Serialization; -using Microsoft.Azure.Functions.Worker; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Azure.Functions.Worker.E2EApp.Blob -{ - public class BlobTestFunctions - { - private readonly ILogger _logger; - - public BlobTestFunctions(ILogger logger) - { - _logger = logger; - } - - [Function(nameof(BlobTriggerToBlobTest))] - [BlobOutput("test-output-dotnet-isolated/{name}")] - public byte[] BlobTriggerToBlobTest( - [BlobTrigger("test-triggerinput-dotnet-isolated/{name}")] byte[] triggerBlob, string name, - [BlobInput("test-input-dotnet-isolated/{name}")] byte[] inputBlob, - FunctionContext context) - { - _logger.LogInformation("Trigger:\n Name: " + name + "\n Size: " + triggerBlob.Length + " Bytes"); - _logger.LogInformation("Input:\n Name: " + name + "\n Size: " + inputBlob.Length + " Bytes"); - return inputBlob; - } - - [Function(nameof(BlobTriggerPocoTest))] - [BlobOutput("test-outputpoco-dotnet-isolated/{name}")] - public TestBlobData BlobTriggerPocoTest( - [BlobTrigger("test-triggerinputpoco-dotnet-isolated/{name}")] TestBlobData triggerBlob, string name, - FunctionContext context) - { - _logger.LogInformation(".NET Blob trigger function processed a blob.\n Name: " + name + "\n Content: " + triggerBlob.BlobText); - return triggerBlob; - } - - [Function(nameof(BlobTriggerStringTest))] - [BlobOutput("test-outputstring-dotnet-isolated/{name}")] - public string BlobTriggerStringTest( - [BlobTrigger("test-triggerinputstring-dotnet-isolated/{name}")] string triggerBlobText, string name, - FunctionContext context) - { - _logger.LogInformation(".NET Blob trigger function processed a blob.\n Name: " + name + "\n Content: " + triggerBlobText); - return triggerBlobText; - } - - public class TestBlobData - { - [JsonPropertyName("text")] - public string BlobText { get; set; } - } - } -} diff --git a/test/E2ETests/E2EApps/E2EApp/Blob/BlobTriggerBindingFunctions.cs b/test/E2ETests/E2EApps/E2EApp/Blob/BlobTriggerBindingFunctions.cs new file mode 100644 index 000000000..ecce43f1a --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/Blob/BlobTriggerBindingFunctions.cs @@ -0,0 +1,90 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.IO; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.Functions.Worker.E2EApp.Blob +{ + public class BlobTriggerBindingFunctions + { + private readonly ILogger _logger; + + public BlobTriggerBindingFunctions(ILogger logger) + { + _logger = logger; + } + + [Function(nameof(BlobTriggerToBlobTest))] + [BlobOutput("test-output-dotnet-isolated/{name}")] + public byte[] BlobTriggerToBlobTest( + [BlobTrigger("test-trigger-dotnet-isolated/{name}")] byte[] triggerBlob, string name, + [BlobInput("test-input-dotnet-isolated/{name}")] byte[] inputBlob, + FunctionContext context) + { + _logger.LogInformation("Trigger:\n Name: " + name + "\n Size: " + triggerBlob.Length + " Bytes"); + _logger.LogInformation("Input:\n Name: " + name + "\n Size: " + inputBlob.Length + " Bytes"); + return inputBlob; + } + + [Function(nameof(BlobTriggerPocoTest))] + [BlobOutput("test-output-poco-dotnet-isolated/{name}")] + public TestBlobData BlobTriggerPocoTest( + [BlobTrigger("test-trigger-poco-dotnet-isolated/{name}")] TestBlobData triggerBlob, string name, + FunctionContext context) + { + _logger.LogInformation(".NET Blob trigger function processed a blob.\n Name: " + name + "\n Content: " + triggerBlob.BlobText); + return triggerBlob; + } + + [Function(nameof(BlobTriggerStringTest))] + [BlobOutput("test-output-string-dotnet-isolated/{name}")] + public string BlobTriggerStringTest( + [BlobTrigger("test-trigger-string-dotnet-isolated/{name}")] string triggerBlobText, string name, + FunctionContext context) + { + _logger.LogInformation(".NET Blob trigger function processed a blob.\n Name: " + name + "\n Content: " + triggerBlobText); + return triggerBlobText; + } + + [Function(nameof(BlobTriggerStreamTest))] + public async Task BlobTriggerStreamTest( + [BlobTrigger("test-trigger-stream-dotnet-isolated/{name}")] Stream stream, string name, + FunctionContext context) + { + using var blobStreamReader = new StreamReader(stream); + string content = await blobStreamReader.ReadToEndAsync(); + _logger.LogInformation("StreamTriggerOutput: {c}", content); + } + + [Function(nameof(BlobTriggerBlobClientTest))] + public async Task BlobTriggerBlobClientTest( + [BlobTrigger("test-trigger-blobclient-dotnet-isolated/{name}")] BlobClient client, string name, + FunctionContext context) + { + var downloadResult = await client.DownloadContentAsync(); + string content = downloadResult.Value.Content.ToString(); + _logger.LogInformation("BlobClientTriggerOutput: {c}", content); + } + + [Function(nameof(BlobTriggerBlobContainerClientTest))] + public async Task BlobTriggerBlobContainerClientTest( + [BlobTrigger("test-trigger-containerclient-dotnet-isolated/{name}")] BlobContainerClient client, string name, + FunctionContext context) + { + var blobClient = client.GetBlobClient(name); + var downloadResult = await blobClient.DownloadContentAsync(); + string content = downloadResult.Value.Content.ToString(); + _logger.LogInformation("BlobContainerTriggerOutput: {c}", content); + } + + public class TestBlobData + { + [JsonPropertyName("text")] + public string BlobText { get; set; } + } + } +} diff --git a/test/E2ETests/E2EApps/E2EApp/Blob/Book.cs b/test/E2ETests/E2EApps/E2EApp/Blob/Book.cs new file mode 100644 index 000000000..2cea88356 --- /dev/null +++ b/test/E2ETests/E2EApps/E2EApp/Blob/Book.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.Functions.Worker.E2EApp.Blob +{ + public class Book + { + public string Id { get; set; } + public string Name { get; set; } + } +} \ No newline at end of file diff --git a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj index f6fd3de04..a170f9c3b 100644 --- a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj +++ b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj @@ -42,9 +42,8 @@ - - - + + diff --git a/test/E2ETests/E2ETests/Constants.cs b/test/E2ETests/E2ETests/Constants.cs index 70256b28a..f9cd316f0 100644 --- a/test/E2ETests/E2ETests/Constants.cs +++ b/test/E2ETests/E2ETests/Constants.cs @@ -34,15 +34,19 @@ public static class Queue //Blob tests public static class Blob { - public const string TriggerInputBindingContainer = "test-triggerinput-dotnet-isolated"; + public const string TriggerInputBindingContainer = "test-trigger-dotnet-isolated"; public const string InputBindingContainer = "test-input-dotnet-isolated"; public const string OutputBindingContainer = "test-output-dotnet-isolated"; - - public const string TriggerPocoContainer = "test-triggerinputpoco-dotnet-isolated"; - public const string OutputPocoContainer = "test-outputpoco-dotnet-isolated"; - - public const string TriggerStringContainer = "test-triggerinputstring-dotnet-isolated"; - public const string OutputStringContainer = "test-outputstring-dotnet-isolated"; + public const string TriggerPocoContainer = "test-trigger-poco-dotnet-isolated"; + public const string OutputPocoContainer = "test-output-poco-dotnet-isolated"; + public const string TriggerStringContainer = "test-trigger-string-dotnet-isolated"; + public const string OutputStringContainer = "test-output-string-dotnet-isolated"; + public const string TriggerStreamContainer = "test-trigger-stream-dotnet-isolated"; + public const string OutputStreamContainer = "test-output-stream-dotnet-isolated"; + public const string TriggerBlobClientContainer = "test-trigger-blobclient-dotnet-isolated"; + public const string OutputBlobClientContainer = "test-output-blobclient-dotnet-isolated"; + public const string TriggerBlobContainerClientContainer = "test-trigger-containerclient-dotnet-isolated"; + public const string OutputBlobContainerClientContainer = "test-output-containerclient-dotnet-isolated"; } // CosmosDB tests diff --git a/test/E2ETests/E2ETests/CosmosDBEndToEndTests.cs b/test/E2ETests/E2ETests/Cosmos/CosmosDBEndToEndTests.cs similarity index 95% rename from test/E2ETests/E2ETests/CosmosDBEndToEndTests.cs rename to test/E2ETests/E2ETests/Cosmos/CosmosDBEndToEndTests.cs index 1332676e9..71fb75934 100644 --- a/test/E2ETests/E2ETests/CosmosDBEndToEndTests.cs +++ b/test/E2ETests/E2ETests/Cosmos/CosmosDBEndToEndTests.cs @@ -6,7 +6,7 @@ using Xunit; using Xunit.Abstractions; -namespace Microsoft.Azure.Functions.Tests.E2ETests +namespace Microsoft.Azure.Functions.Tests.E2ETests.Cosmos { [Collection(Constants.FunctionAppCollectionName)] public class CosmosDBEndToEndTests : IDisposable diff --git a/test/E2ETests/E2ETests/Helpers/StorageHelpers.cs b/test/E2ETests/E2ETests/Helpers/StorageHelpers.cs index 191717b48..f710ebcd7 100644 --- a/test/E2ETests/E2ETests/Helpers/StorageHelpers.cs +++ b/test/E2ETests/E2ETests/Helpers/StorageHelpers.cs @@ -108,6 +108,12 @@ public async static Task ClearBlobContainers() await ClearBlobContainer(Constants.Blob.OutputPocoContainer); await ClearBlobContainer(Constants.Blob.TriggerStringContainer); await ClearBlobContainer(Constants.Blob.OutputStringContainer); + await ClearBlobContainer(Constants.Blob.TriggerStreamContainer); + await ClearBlobContainer(Constants.Blob.OutputStreamContainer); + await ClearBlobContainer(Constants.Blob.TriggerBlobClientContainer); + await ClearBlobContainer(Constants.Blob.OutputBlobClientContainer); + await ClearBlobContainer(Constants.Blob.TriggerBlobContainerClientContainer); + await ClearBlobContainer(Constants.Blob.OutputBlobContainerClientContainer); } public static Task UploadFileToContainer(string containerName, string fileName) diff --git a/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs b/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs new file mode 100644 index 000000000..a3f41d63c --- /dev/null +++ b/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs @@ -0,0 +1,292 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Azure.Functions.Tests.E2ETests.Storage +{ + [Collection(Constants.FunctionAppCollectionName)] + public class BlobEndToEndTests : IDisposable + { + private readonly IDisposable _disposeLog; + private FunctionAppFixture _fixture; + + public BlobEndToEndTests(FunctionAppFixture fixture, ITestOutputHelper testOutput) + { + _fixture = fixture; + _disposeLog = _fixture.TestLogs.UseTestLogger(testOutput); + } + + [Fact] + public async Task BlobTriggerToBlob_Succeeds() + { + string fileName = Guid.NewGuid().ToString(); + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Setup + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, fileName); + + //Trigger + await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerInputBindingContainer, fileName); + + //Verify + string result = await StorageHelpers.DownloadFileFromContainer(Constants.Blob.OutputBindingContainer, fileName); + + Assert.Equal("Hello World", result); + } + + [Fact] + public async Task BlobTrigger_Poco_Succeeds() + { + string fileName = Guid.NewGuid().ToString(); + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Trigger + var json = JsonSerializer.Serialize(new { text = "Hello World" }); + await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerPocoContainer, fileName, json); + + //Verify + string result = await StorageHelpers.DownloadFileFromContainer(Constants.Blob.OutputPocoContainer, fileName); + + Assert.Equal(json, result); + } + + [Fact] + public async Task BlobTrigger_String_Succeeds() + { + string fileName = Guid.NewGuid().ToString(); + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Trigger + await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerStringContainer, fileName); + + //Verify + string result = await StorageHelpers.DownloadFileFromContainer(Constants.Blob.OutputStringContainer, fileName); + + Assert.Equal("Hello World", result); + } + + [Fact] + public async Task BlobTrigger_Stream_Succeeds() + { + string key = "StreamTriggerOutput: "; + string fileName = Guid.NewGuid().ToString(); + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Trigger + await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerStreamContainer, fileName); + + //Verify + IEnumerable logs = null; + await TestUtility.RetryAsync(() => + { + logs = _fixture.TestLogs.CoreToolsLogs.Where(p => p.Contains(key)); + return Task.FromResult(logs.Count() >= 1); + }); + + var lastLog = logs.Last(); + int subStringStart = lastLog.LastIndexOf(key) + key.Length; + var result = lastLog[subStringStart..]; + + Assert.Equal("Hello World", result); + } + + [Fact] + public async Task BlobTrigger_BlobClient_Succeeds() + { + string key = "BlobClientTriggerOutput: "; + string fileName = Guid.NewGuid().ToString(); + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Trigger + await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerBlobClientContainer, fileName); + + //Verify + IEnumerable logs = null; + await TestUtility.RetryAsync(() => + { + logs = _fixture.TestLogs.CoreToolsLogs.Where(p => p.Contains(key)); + return Task.FromResult(logs.Count() >= 1); + }); + + var lastLog = logs.Last(); + int subStringStart = lastLog.LastIndexOf(key) + key.Length; + var result = lastLog[subStringStart..]; + + Assert.Equal("Hello World", result); + } + + [Fact] + public async Task BlobTrigger_BlobContainerClient_Succeeds() + { + string key = "BlobContainerTriggerOutput: "; + string fileName = Guid.NewGuid().ToString(); + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Trigger + await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerBlobContainerClientContainer, fileName); + + //Verify + IEnumerable logs = null; + await TestUtility.RetryAsync(() => + { + logs = _fixture.TestLogs.CoreToolsLogs.Where(p => p.Contains(key)); + return Task.FromResult(logs.Count() >= 1); + }); + + var lastLog = logs.Last(); + int subStringStart = lastLog.LastIndexOf(key) + key.Length; + var result = lastLog[subStringStart..]; + + Assert.Equal("Hello World", result); + } + + [Theory] + [InlineData("BlobInputClientTest")] + [InlineData("BlobInputContainerClientTest")] + [InlineData("BlobInputStreamTest")] + [InlineData("BlobInputByteTest")] + [InlineData("BlobInputStringTest")] + public async Task BlobInput_SingleCardinality_Succeeds(string functionName) + { + string expectedMessage = "Hello World"; + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Setup + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile", expectedMessage); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionName); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Contains(expectedMessage, actualMessage); + } + + [Fact] + public async Task BlobInput_Poco_Succeeds() + { + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Setup + var json = JsonSerializer.Serialize(new { id = "1", name = "To Kill a Mockingbird" }); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile", json); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputPocoTest"); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + string expectedMessage = "To Kill a Mockingbird"; + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Contains(expectedMessage, actualMessage); + } + + [Fact(Skip = "Collection support released in host version 4.16+")] + public async Task BlobInput_BlobClientCollection_Succeeds() + { + string expectedMessage = "testFile1, testFile2, testFile3"; + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Setup + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile1"); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile2"); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile3"); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputCollectionTest"); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Contains(expectedMessage, actualMessage); + } + + [Fact(Skip = "Collection support released in host version 4.16+")] + public async Task BlobInput_StringCollection_Succeeds() + { + string expectedMessage = "ABC, DEF, GHI"; + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Setup + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile1", "ABC"); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile2", "DEF"); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile3", "GHI"); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputCollectionTest"); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Contains(expectedMessage, actualMessage); + } + + [Fact(Skip = "Collection support released in host version 4.16+")] + public async Task BlobInput_PocoCollection_Succeeds() + { + string book1 = $@"{{ ""id"": ""1"", ""name"": ""To Kill a Mockingbird""}}"; + string book2 = $@"{{ ""id"": ""2"", ""name"": ""Of Mice and Men""}}"; + string book3 = $@"{{ ""id"": ""3"", ""name"": ""The Wind in the Willows""}}"; + + string expectedMessage = "To Kill a Mockingbird, Of Mice and Men, The Wind in the Willows"; + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Setup + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "book1", book1); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "book2", book2); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "book3", book3); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputPocoArrayTest"); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Contains(expectedMessage, actualMessage); + } + + public void Dispose() + { + _disposeLog?.Dispose(); + } + } +} diff --git a/test/E2ETests/E2ETests/StorageEndToEndTests.cs b/test/E2ETests/E2ETests/Storage/QueueEndToEndTests.cs similarity index 76% rename from test/E2ETests/E2ETests/StorageEndToEndTests.cs rename to test/E2ETests/E2ETests/Storage/QueueEndToEndTests.cs index 8271aad32..ed81d8cfa 100644 --- a/test/E2ETests/E2ETests/StorageEndToEndTests.cs +++ b/test/E2ETests/E2ETests/Storage/QueueEndToEndTests.cs @@ -9,14 +9,14 @@ using System.Threading.Tasks; using Xunit; -namespace Microsoft.Azure.Functions.Tests.E2ETests +namespace Microsoft.Azure.Functions.Tests.E2ETests.Storage { [Collection(Constants.FunctionAppCollectionName)] - public class StorageEndToEndTests + public class QueueEndToEndTests { private FunctionAppFixture _fixture; - public StorageEndToEndTests(FunctionAppFixture fixture) + public QueueEndToEndTests(FunctionAppFixture fixture) { _fixture = fixture; } @@ -29,7 +29,7 @@ public async Task QueueTriggerAndOutput_Succeeds() await StorageHelpers.ClearQueue(Constants.Queue.OutputBindingName); await StorageHelpers.ClearQueue(Constants.Queue.InputBindingName); - //Set up and trigger + //Set up and trigger await StorageHelpers.CreateQueue(Constants.Queue.OutputBindingName); await StorageHelpers.InsertIntoQueue(Constants.Queue.InputBindingName, expectedQueueMessage); @@ -46,7 +46,7 @@ public async Task QueueTriggerAndArrayOutput_Succeeds() await StorageHelpers.ClearQueue(Constants.Queue.InputArrayBindingName); await StorageHelpers.ClearQueue(Constants.Queue.OutputArrayBindingName); - //Set up and trigger + //Set up and trigger await StorageHelpers.CreateQueue(Constants.Queue.OutputArrayBindingName); await StorageHelpers.InsertIntoQueue(Constants.Queue.InputArrayBindingName, expectedQueueMessage); @@ -72,7 +72,7 @@ public async Task QueueTriggerAndListOutput_Succeeds() await StorageHelpers.ClearQueue(Constants.Queue.InputListBindingName); await StorageHelpers.ClearQueue(Constants.Queue.OutputListBindingName); - //Set up and trigger + //Set up and trigger await StorageHelpers.CreateQueue(Constants.Queue.OutputListBindingName); await StorageHelpers.InsertIntoQueue(Constants.Queue.InputListBindingName, expectedQueueMessage); @@ -98,7 +98,7 @@ public async Task QueueTriggerAndBindingDataOutput_Succeeds() await StorageHelpers.ClearQueue(Constants.Queue.InputBindingDataName); await StorageHelpers.ClearQueue(Constants.Queue.OutputBindingDataName); - //Set up and trigger + //Set up and trigger await StorageHelpers.CreateQueue(Constants.Queue.OutputBindingDataName); await StorageHelpers.InsertIntoQueue(Constants.Queue.InputBindingDataName, expectedQueueMessage); @@ -122,7 +122,7 @@ public async Task QueueTrigger_BindToTriggerMetadata_Succeeds() await StorageHelpers.ClearQueue(Constants.Queue.OutputBindingNameMetadata); await StorageHelpers.ClearQueue(Constants.Queue.InputBindingNameMetadata); - //Set up and trigger + //Set up and trigger await StorageHelpers.CreateQueue(Constants.Queue.OutputBindingNameMetadata); string expectedQueueMessage = await StorageHelpers.InsertIntoQueue(Constants.Queue.InputBindingNameMetadata, inputQueueMessage); @@ -139,7 +139,7 @@ public async Task QueueTrigger_QueueOutput_Poco_Succeeds() await StorageHelpers.ClearQueue(Constants.Queue.OutputBindingNamePOCO); await StorageHelpers.ClearQueue(Constants.Queue.InputBindingNamePOCO); - //Set up and trigger + //Set up and trigger await StorageHelpers.CreateQueue(Constants.Queue.OutputBindingNamePOCO); string json = JsonSerializer.Serialize(new { id = expectedQueueMessage }); @@ -166,60 +166,5 @@ public async Task QueueOutput_PocoList_Succeeds() IEnumerable queueMessages = await StorageHelpers.ReadMessagesFromQueue(Constants.Queue.OutputBindingNamePOCO); Assert.True(queueMessages.All(msg => msg.Contains(expectedQueueMessage))); } - - [Fact] - public async Task BlobTriggerToBlob_Succeeds() - { - string fileName = Guid.NewGuid().ToString(); - - //cleanup - await StorageHelpers.ClearBlobContainers(); - - //Setup - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, fileName); - - //Trigger - await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerInputBindingContainer, fileName); - - //Verify - string result = await StorageHelpers.DownloadFileFromContainer(Constants.Blob.OutputBindingContainer, fileName); - - Assert.Equal("Hello World", result); - } - - [Fact] - public async Task BlobTriggerPoco_Succeeds() - { - string fileName = Guid.NewGuid().ToString(); - - //cleanup - await StorageHelpers.ClearBlobContainers(); - - //Trigger - var json = JsonSerializer.Serialize(new { text = "Hello World" }); - await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerPocoContainer, fileName, json); - - //Verify - string result = await StorageHelpers.DownloadFileFromContainer(Constants.Blob.OutputPocoContainer, fileName); - - Assert.Equal(json, result); - } - - [Fact] - public async Task BlobTriggerString_Succeeds() - { - string fileName = Guid.NewGuid().ToString(); - - //cleanup - await StorageHelpers.ClearBlobContainers(); - - //Trigger - await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerStringContainer, fileName); - - //Verify - string result = await StorageHelpers.DownloadFileFromContainer(Constants.Blob.OutputStringContainer, fileName); - - Assert.Equal("Hello World", result); - } } } From 805b2566b635fdc9b808dcd9f832dd638b9466fb Mon Sep 17 00:00:00 2001 From: Surgupta Date: Thu, 23 Feb 2023 15:00:47 -0800 Subject: [PATCH 15/47] Added unit tests for BlobStorageConverter (#1370) --- .../src/BlobStorageConverter.cs | 34 +- .../Config/BlobStorageBindingOptionsSetup.cs | 1 + .../src/Constants.cs | 2 +- .../src/Properties/AssemblyInfo.cs | 4 + .../WorkerBindingSamples/{ => Blob}/Book.cs | 0 .../Blob/BlobStorageConverterTests.cs | 413 ++++++++++++++++++ .../Worker.Extensions.Tests.csproj | 5 + 7 files changed, 441 insertions(+), 18 deletions(-) rename samples/WorkerBindingSamples/{ => Blob}/Book.cs (100%) create mode 100644 test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs index 3e0a6c96f..280ccc22e 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -12,6 +12,7 @@ using Azure.Storage.Blobs.Specialized; using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs; using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging; @@ -24,7 +25,6 @@ internal class BlobStorageConverter : IInputConverter { private readonly IOptions _workerOptions; private readonly IOptionsSnapshot _blobOptions; - private readonly ILogger _logger; public BlobStorageConverter(IOptions workerOptions, IOptionsSnapshot blobOptions, ILogger logger) @@ -44,7 +44,7 @@ public async ValueTask ConvertAsync(ConverterContext context) }; } - private async ValueTask ConvertFromBindingDataAsync(ConverterContext context, ModelBindingData modelBindingData) + internal virtual async ValueTask ConvertFromBindingDataAsync(ConverterContext context, ModelBindingData modelBindingData) { if (!IsBlobExtension(modelBindingData)) { @@ -69,7 +69,7 @@ private async ValueTask ConvertFromBindingDataAsync(ConverterC return ConversionResult.Unhandled(); } - private async ValueTask ConvertFromCollectionBindingDataAsync(ConverterContext context, CollectionModelBindingData collectionModelBindingData) + internal virtual async ValueTask ConvertFromCollectionBindingDataAsync(ConverterContext context, CollectionModelBindingData collectionModelBindingData) { var blobCollection = new List(collectionModelBindingData.ModelBindingDataArray.Length); Type elementType = context.TargetType.IsArray ? context.TargetType.GetElementType() : context.TargetType.GenericTypeArguments[0]; @@ -103,7 +103,7 @@ private async ValueTask ConvertFromCollectionBindingDataAsync( } } - private bool IsBlobExtension(ModelBindingData bindingData) + internal bool IsBlobExtension(ModelBindingData bindingData) { if (bindingData?.Source is not Constants.BlobExtensionName) { @@ -114,7 +114,7 @@ private bool IsBlobExtension(ModelBindingData bindingData) return true; } - private Dictionary GetBindingDataContent(ModelBindingData bindingData) + internal Dictionary GetBindingDataContent(ModelBindingData bindingData) { return bindingData?.ContentType switch { @@ -123,7 +123,7 @@ private Dictionary GetBindingDataContent(ModelBindingData bindin }; } - private async Task ConvertModelBindingDataAsync(IDictionary content, Type targetType, ModelBindingData bindingData) + internal virtual async Task ConvertModelBindingDataAsync(IDictionary content, Type targetType, ModelBindingData bindingData) { content.TryGetValue(Constants.Connection, out var connectionName); content.TryGetValue(Constants.ContainerName, out var containerName); @@ -137,7 +137,7 @@ private Dictionary GetBindingDataContent(ModelBindingData bindin return await ToTargetTypeAsync(targetType, connectionName, containerName, blobName); } - private async Task ToTargetTypeAsync(Type targetType, string connectionName, string containerName, string blobName) => targetType switch + internal virtual async Task ToTargetTypeAsync(Type targetType, string connectionName, string containerName, string blobName) => targetType switch { Type _ when targetType == typeof(String) => await GetBlobStringAsync(connectionName, containerName, blobName), Type _ when targetType == typeof(Stream) => await GetBlobStreamAsync(connectionName, containerName, blobName), @@ -151,13 +151,13 @@ private Dictionary GetBindingDataContent(ModelBindingData bindin _ => await DeserializeToTargetObjectAsync(targetType, connectionName, containerName, blobName) }; - private async Task DeserializeToTargetObjectAsync(Type targetType, string connectionName, string containerName, string blobName) + internal async Task DeserializeToTargetObjectAsync(Type targetType, string connectionName, string containerName, string blobName) { var content = await GetBlobStreamAsync(connectionName, containerName, blobName); return _workerOptions?.Value?.Serializer?.Deserialize(content, targetType, CancellationToken.None); } - private object? ToTargetTypeCollection(IEnumerable blobCollection, string methodName, Type type) + internal object? ToTargetTypeCollection(IEnumerable blobCollection, string methodName, Type type) { blobCollection = blobCollection.Select(b => Convert.ChangeType(b, type)); MethodInfo method = typeof(BlobStorageConverter).GetMethod(methodName, BindingFlags.Static | BindingFlags.NonPublic); @@ -166,17 +166,17 @@ private Dictionary GetBindingDataContent(ModelBindingData bindin return genericMethod.Invoke(null, new[] { blobCollection.ToList() }); } - private static T[] CloneToArray(IList source) + internal static T[] CloneToArray(IList source) { return source.Cast().ToArray(); } - private static IEnumerable CloneToList(IList source) + internal static IEnumerable CloneToList(IList source) { return source.Cast(); } - private async Task GetBlobStringAsync(string connectionName, string containerName, string blobName) + internal virtual async Task GetBlobStringAsync(string connectionName, string containerName, string blobName) { var client = CreateBlobClient(connectionName, containerName, blobName); return await GetBlobContentStringAsync(client); @@ -188,7 +188,7 @@ private async Task GetBlobContentStringAsync(BlobClient client) return download.Value.Content.ToString(); } - private async Task GetBlobBinaryDataAsync(string connectionName, string containerName, string blobName) + internal virtual async Task GetBlobBinaryDataAsync(string connectionName, string containerName, string blobName) { using MemoryStream stream = new(); var client = CreateBlobClient(connectionName, containerName, blobName); @@ -196,14 +196,14 @@ private async Task GetBlobBinaryDataAsync(string connectionName, string return stream.ToArray(); } - private async Task GetBlobStreamAsync(string connectionName, string containerName, string blobName) + internal virtual async Task GetBlobStreamAsync(string connectionName, string containerName, string blobName) { var client = CreateBlobClient(connectionName, containerName, blobName); var download = await client.DownloadStreamingAsync(); return download.Value.Content; } - private BlobContainerClient CreateBlobContainerClient(string connectionName, string containerName) + internal virtual BlobContainerClient CreateBlobContainerClient(string connectionName, string containerName) { var blobStorageOptions = _blobOptions.Get(connectionName); BlobServiceClient blobServiceClient = blobStorageOptions.CreateClient(); @@ -211,7 +211,7 @@ private BlobContainerClient CreateBlobContainerClient(string connectionName, str return container; } - private T CreateBlobClient(string connectionName, string containerName, string blobName) where T : BlobBaseClient + internal virtual T CreateBlobClient(string connectionName, string containerName, string blobName) where T : BlobBaseClient { if (string.IsNullOrEmpty(blobName)) { @@ -233,4 +233,4 @@ private T CreateBlobClient(string connectionName, string containerName, strin return (T)blobClient; } } -} \ No newline at end of file +} diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptionsSetup.cs b/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptionsSetup.cs index b4c538042..654653702 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptionsSetup.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptionsSetup.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Options; using Microsoft.Azure.Functions.Worker.Extensions; +using Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs; using System.Globalization; namespace Microsoft.Azure.Functions.Worker diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Constants.cs b/extensions/Worker.Extensions.Storage.Blobs/src/Constants.cs index 95b378704..5320bff21 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Constants.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Constants.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -namespace Microsoft.Azure.Functions.Worker +namespace Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs { internal static class Constants { diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Properties/AssemblyInfo.cs b/extensions/Worker.Extensions.Storage.Blobs/src/Properties/AssemblyInfo.cs index 14ae3e451..7bd16b23e 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Properties/AssemblyInfo.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Properties/AssemblyInfo.cs @@ -1,6 +1,10 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System.Runtime.CompilerServices; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; [assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.2")] +[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] + diff --git a/samples/WorkerBindingSamples/Book.cs b/samples/WorkerBindingSamples/Blob/Book.cs similarity index 100% rename from samples/WorkerBindingSamples/Book.cs rename to samples/WorkerBindingSamples/Blob/Book.cs diff --git a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs new file mode 100644 index 000000000..903d7a7f1 --- /dev/null +++ b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs @@ -0,0 +1,413 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; +using Google.Protobuf; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Grpc.Messages; +using Microsoft.Azure.Functions.Worker.Tests; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +using Constants = Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs.Constants; + +namespace Microsoft.Azure.Functions.WorkerExtension.Tests +{ + public class BlobStorageConverterTests + { + private Mock _mockBlobStorageConverter; + + public BlobStorageConverterTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + var workerOptions = host.Services.GetService>(); + var blobOptions = host.Services.GetService>(); + var logger = host.Services.GetService>(); + + _mockBlobStorageConverter = new Mock(workerOptions, blobOptions, logger); + _mockBlobStorageConverter.CallBase = true; + } + + [Fact] + public async Task ConvertAsync_SourceAsObject_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(string), new Object()); + + var conversionResult = await _mockBlobStorageConverter.Object.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_SourceAsModelBindingData_ReturnsSuccess() + { + object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + var context = new TestConverterContext(typeof(string), source); + _mockBlobStorageConverter + .Setup(c => c.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName)) + .ReturnsAsync("test"); + + var conversionResult = await _mockBlobStorageConverter.Object.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_SourceAsCollectionModelBindingData_ReturnsSuccess() + { + object source = GetTestGrpcCollectionModelBindingData(); + var context = new TestConverterContext(typeof(string), source); + _mockBlobStorageConverter + .Setup(c => c.ConvertFromCollectionBindingDataAsync(context, (Worker.Core.CollectionModelBindingData) source)) + .Returns(new ValueTask(ConversionResult.Success("test"))); + + var conversionResult = await _mockBlobStorageConverter.Object.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_SourceAsModelBindingData_ReturnsFailure() + { + object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + var context = new TestConverterContext(typeof(string), source); + _mockBlobStorageConverter + .Setup(c => c.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName)) + .ThrowsAsync(new Exception()); + + var conversionResult = await _mockBlobStorageConverter.Object.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertFromBindingDataAsync_ReturnsSuccess() + { + object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + var contentDict = GetTestContentDict(); + var context = new TestConverterContext(typeof(string), source); + var modelBindingData = (Worker.Core.ModelBindingData) source; + + _mockBlobStorageConverter + .Setup(c => c.ConvertModelBindingDataAsync(contentDict, typeof(string), modelBindingData)) + .ReturnsAsync(new ValueTask(ConversionResult.Success("test"))); + + var conversionResult = await _mockBlobStorageConverter.Object.ConvertFromBindingDataAsync(context, modelBindingData); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + } + + [Fact] + public async Task ConvertFromBindingDataAsync_ReturnsFailure() + { + object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + var contentDict = GetTestContentDict(); + var context = new TestConverterContext(typeof(string), source); + + _mockBlobStorageConverter + .Setup(c => c.ConvertModelBindingDataAsync(contentDict, typeof(string), (Worker.Core.ModelBindingData)source)) + .ThrowsAsync(new Exception()); + + var conversionResult = await _mockBlobStorageConverter.Object.ConvertFromBindingDataAsync(context, (Worker.Core.ModelBindingData)source); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertFromBindingDataAsync_ReturnsUnhandled() + { + object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + var context = new TestConverterContext(typeof(string), source); + var dict = _mockBlobStorageConverter.Object.GetBindingDataContent((Worker.Core.ModelBindingData)source); + + _mockBlobStorageConverter.Setup(c => c.ConvertModelBindingDataAsync(dict, typeof(string), (Worker.Core.ModelBindingData)source)).ReturnsAsync(null); + + var conversionResult = await _mockBlobStorageConverter.Object.ConvertFromBindingDataAsync(context, (Worker.Core.ModelBindingData)source); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Theory] + [InlineData(Constants.BlobExtensionName, true)] + [InlineData(" ", false)] + [InlineData("incorrect-value", false)] + public void IsBlobExtension_MatchesExpectedOutput(string sourceVal, bool expectedResult) + { + var grpcModelBindingData = new GrpcModelBindingData(new ModelBindingData() + { + Version = "1.0", + Source = sourceVal, + Content = ByteString.CopyFrom(GetFullTestBinaryData()), + ContentType = "application/json" + }); + + var result = _mockBlobStorageConverter.Object.IsBlobExtension(grpcModelBindingData); + + Assert.Equal(result, expectedResult); + } + + [Fact] + public void GetBindingDataContent_CompleteGrpcModelBindingData_Works() + { + var grpcModelBindingData = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + var result = _mockBlobStorageConverter.Object.GetBindingDataContent(grpcModelBindingData); + + result.TryGetValue(Constants.Connection, out var connectionName); + result.TryGetValue(Constants.ContainerName, out var containerName); + result.TryGetValue(Constants.BlobName, out var blobName); + + Assert.Equal(3, result.Count); + Assert.Equal(Constants.Connection, connectionName); + Assert.Equal(Constants.ContainerName, containerName); + Assert.Equal(Constants.BlobName, blobName); + } + + [Fact] + public void GetBindingDataContent_IncompleteGrpcModelBindingData_ReturnsNull() + { + var grpcModelBindingData = GetTestGrpcModelBindingData(GetTestBinaryData()); + + var result = _mockBlobStorageConverter.Object.GetBindingDataContent(grpcModelBindingData); + + result.TryGetValue(Constants.Connection, out var connectionName); + result.TryGetValue(Constants.ContainerName, out var containerName); + result.TryGetValue(Constants.BlobName, out var blobName); + + Assert.Single(result); + Assert.True(connectionName is null); + Assert.True(containerName is null); + Assert.Equal(Constants.BlobName, blobName); + } + + [Fact] + public void GetBindingDataContent_UnSupportedContentType_Throws() + { + var grpcModelBindingData = new GrpcModelBindingData(new ModelBindingData() + { + Version = "1.0", + Source = Constants.BlobExtensionName, + Content = ByteString.CopyFrom(GetFullTestBinaryData()), + ContentType = "NotSupported" + }); + + try + { + var dict = _mockBlobStorageConverter.Object.GetBindingDataContent(grpcModelBindingData); + Assert.Fail("Test fails as the expected exception not thrown"); + } + catch (Exception ex) + { + Assert.Equal(typeof(NotSupportedException), ex.GetType()); + } + } + + [Fact] + public async Task ConvertModelBindingDataAsync_IncompleteGrpcModelBindingData_Throws() + { + var grpcModelBindingData = GetTestGrpcModelBindingData(GetTestBinaryData()); + var dict = _mockBlobStorageConverter.Object.GetBindingDataContent(grpcModelBindingData); + _mockBlobStorageConverter.Setup(c => c.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync("test"); + + try + { + var result = await _mockBlobStorageConverter.Object.ConvertModelBindingDataAsync(dict, typeof(string), grpcModelBindingData); + Assert.Fail("Test fails as the expected exception not thrown"); + } + catch (ArgumentNullException) { } + } + + [Fact] + public async Task ConvertModelBindingDataAsync_GrpcModelBindingData_Works() + { + var grpcModelBindingData = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + var contentDict = GetTestContentDict(); + + _mockBlobStorageConverter + .Setup(c => c.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName)) + .ReturnsAsync("test"); + + var result = await _mockBlobStorageConverter.Object.ConvertModelBindingDataAsync(contentDict, typeof(string), grpcModelBindingData); + + Assert.Equal(typeof(string), result.GetType()); + + } + + [Fact] + public async Task ToTargetTypeAsync_Works() + { + object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + byte[] byteArray = Encoding.UTF8.GetBytes("test"); + + _mockBlobStorageConverter.Setup(c => c.GetBlobStreamAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync(new MemoryStream(byteArray)); + _mockBlobStorageConverter.Setup(c => c.GetBlobBinaryDataAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync(byteArray); + _mockBlobStorageConverter.Setup(c => c.GetBlobStringAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync("test"); + _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); + _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); + _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); + _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); + _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); + _mockBlobStorageConverter.Setup(c => c.CreateBlobContainerClient(Constants.Connection, Constants.ContainerName)).Returns(new Mock().Object); + + var streamResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(Stream), Constants.Connection, Constants.ContainerName, Constants.BlobName); + var byteArrayResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(Byte[]), Constants.Connection, Constants.ContainerName, Constants.BlobName); + var stringResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName); + var blobClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(BlobClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); + var blobBaseClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(BlobBaseClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); + var blockBlobClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(BlockBlobClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); + var pageBlobClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(PageBlobClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); + var appendBlobClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(AppendBlobClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); + var blobContainerClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(BlobContainerClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); + + Assert.Equal(typeof(MemoryStream), streamResult.GetType()); + Assert.Equal(typeof(Byte[]), byteArrayResult.GetType()); + Assert.Equal(typeof(string), stringResult.GetType()); + Assert.Equal(typeof(BlobClient), blobClientResult.GetType().BaseType); + Assert.Equal(typeof(BlobBaseClient), blobBaseClientResult.GetType().BaseType); + Assert.Equal(typeof(BlockBlobClient), blockBlobClientResult.GetType().BaseType); + Assert.Equal(typeof(PageBlobClient), pageBlobClientResult.GetType().BaseType); + Assert.Equal(typeof(AppendBlobClient), appendBlobClientResult.GetType().BaseType); + Assert.Equal(typeof(BlobContainerClient), blobContainerClientResult.GetType().BaseType); + } + + [Fact] + public void ToTargetTypeCollection_CloneToArray_Works() + { + IEnumerable stringCollection = new List() { "hello", "world"}; + string[] stringResult = (string[])_mockBlobStorageConverter.Object.ToTargetTypeCollection(stringCollection, nameof(BlobStorageConverter.CloneToArray), typeof(string)); + + IEnumerable pocoCollection = new List() { new Book(), new Book() }; + Book[] pocoResult = (Book[])_mockBlobStorageConverter.Object.ToTargetTypeCollection(pocoCollection, nameof(BlobStorageConverter.CloneToArray), typeof(Book)); + + IEnumerable byteArraycollection = new List() { Encoding.UTF8.GetBytes("hello"), Encoding.UTF8.GetBytes("world") }; + Byte[][] byteArrayResult = (Byte[][])_mockBlobStorageConverter.Object.ToTargetTypeCollection(byteArraycollection, nameof(BlobStorageConverter.CloneToArray), typeof(Byte[])); + + Assert.Equal(2, stringResult.Length); + Assert.Equal(typeof(string), stringResult[0].GetType()); + + Assert.Equal(2, pocoResult.Length); + Assert.Equal(typeof(Book), pocoResult[0].GetType()); + + Assert.Equal(2, byteArrayResult.Length); + Assert.Equal(typeof(Byte[]), byteArrayResult[0].GetType()); + } + + [Fact] + public void ToTargetTypeCollection_CloneToList_Works() + { + IEnumerable stringCollection = new List() { "hello", "world" }; + IEnumerable stringResult = (IEnumerable)_mockBlobStorageConverter.Object.ToTargetTypeCollection(stringCollection, nameof(BlobStorageConverter.CloneToArray), typeof(string)); + + IEnumerable pocoCollection = new List() { new Book(), new Book() }; + IEnumerable pocoResult = (IEnumerable)_mockBlobStorageConverter.Object.ToTargetTypeCollection(pocoCollection, nameof(BlobStorageConverter.CloneToList), typeof(Book)); + + IEnumerable byteArraycollection = new List() { Encoding.UTF8.GetBytes("hello"), Encoding.UTF8.GetBytes("world") }; + IEnumerable byteArrayResult = (IEnumerable)_mockBlobStorageConverter.Object.ToTargetTypeCollection(byteArraycollection, nameof(BlobStorageConverter.CloneToArray), typeof(Byte[])); + + Assert.Equal(2, stringResult.Count()); + Assert.Equal(typeof(string), stringResult.FirstOrDefault().GetType()); + + Assert.Equal(2, pocoResult.Count()); + Assert.Equal(typeof(Book), pocoResult.FirstOrDefault().GetType()); + + Assert.Equal(2, byteArrayResult.Count()); + Assert.Equal(typeof(Byte[]), byteArrayResult.FirstOrDefault().GetType()); + } + + + [Fact] + public async Task DeserializeToTargetObjectAsync_CorrectPoco_Works() + { + object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + string jsonstr = "{" + "\"Id\" : \"1\", \"Title\" : \"title\", \"Author\" : \"author\"}"; + byte[] byteArray = Encoding.UTF8.GetBytes(jsonstr); + + _mockBlobStorageConverter.Setup(c => c.GetBlobStreamAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync(new MemoryStream(byteArray)); + + var result = await _mockBlobStorageConverter.Object.DeserializeToTargetObjectAsync(typeof(Book), Constants.Connection, Constants.ContainerName, Constants.BlobName); + + Assert.Equal(typeof(Book), result.GetType()); + } + + [Fact] + public async Task DeserializeToTargetObjectAsync_IncorrectPoco_Fails() + { + object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); + string jsonstr = "{" + "\"Id\" : \"1\", \"Name\" : \"name\"}"; + byte[] byteArray = Encoding.UTF8.GetBytes(jsonstr); + + _mockBlobStorageConverter.Setup(c => c.GetBlobStreamAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync(new MemoryStream(byteArray)); + + try + { + var result = await _mockBlobStorageConverter.Object.DeserializeToTargetObjectAsync(typeof(Book), Constants.Connection, Constants.ContainerName, Constants.BlobName); + Assert.Fail("Test fails as the expected exception not thrown"); + } + catch (Xunit.Sdk.FailException) { } + } + + private BinaryData GetTestBinaryData() + { + return new BinaryData("{" + "\"BlobName\" : \"BlobName\"" + "}"); + } + + private BinaryData GetFullTestBinaryData() + { + return new BinaryData("{" + + "\"Connection\" : \"Connection\"," + + "\"ContainerName\" : \"ContainerName\"," + + "\"BlobName\" : \"BlobName\"" + + "}"); + } + + + private GrpcModelBindingData GetTestGrpcModelBindingData(BinaryData binaryData) + { + return new GrpcModelBindingData(new ModelBindingData() + { + Version = "1.0", + Source = "AzureStorageBlobs", + Content = ByteString.CopyFrom(binaryData), + ContentType = "application/json" + }); + } + + private GrpcCollectionModelBindingData GetTestGrpcCollectionModelBindingData() + { + var modelBindingData = new ModelBindingData() + { + Version = "1.0", + Source = "AzureStorageBlobs", + Content = ByteString.CopyFrom(GetFullTestBinaryData()), + ContentType = "application/json" + }; + + var array = new CollectionModelBindingData(); + array.ModelBindingData.Add(modelBindingData); + + return new GrpcCollectionModelBindingData(array); + } + + private Dictionary GetTestContentDict() + { + return new Dictionary + { + { Constants.Connection, Constants.Connection }, + { Constants.ContainerName, Constants.ContainerName }, + { Constants.BlobName, Constants.BlobName } + }; + } + } +} diff --git a/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj b/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj index 2406459df..6818c4767 100644 --- a/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj +++ b/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj @@ -21,4 +21,9 @@ + + + + + From 6737c18c2abfcc6ccdc95ea452a86dc8e7cb2db0 Mon Sep 17 00:00:00 2001 From: Surgupta Date: Fri, 12 May 2023 15:19:14 -0700 Subject: [PATCH 16/47] Update blob extension deffered binding attributes (#1462) --- .../src/BlobInputAttribute.cs | 4 +- .../src/BlobStorageConverter.cs | 2 + .../src/BlobTriggerAttribute.cs | 4 +- .../src/StorageExtensionStartup.cs | 5 - .../DotNetWorkerTests.csproj | 1 + .../GrpcFunctionDefinitionTests.cs | 84 ++ .../FunctionMetadataGeneratorTests.cs | 226 ++++- .../IntegratedTriggersAndBindingsTests.cs | 37 - .../functions.metadata | 792 ++++++++++++++++++ test/SdkE2ETests/PublishTests.cs | 61 +- test/SdkE2ETests/SdkE2ETests.csproj | 2 + 11 files changed, 1165 insertions(+), 53 deletions(-) create mode 100644 test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs index 78ab31abc..89e4ca58c 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs @@ -1,11 +1,13 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; namespace Microsoft.Azure.Functions.Worker { - [SupportsDeferredBinding] + [AllowConverterFallback(false)] + [InputConverter(typeof(BlobStorageConverter))] public sealed class BlobInputAttribute : InputBindingAttribute { private readonly string _blobPath; diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs index 280ccc22e..b0c4be9f0 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -15,12 +15,14 @@ using Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs; using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; namespace Microsoft.Azure.Functions.Worker { /// /// Converter to bind Blob Storage type parameters. /// + [SupportsDeferredBinding] internal class BlobStorageConverter : IInputConverter { private readonly IOptions _workerOptions; diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs index 59ac9fbc8..6313dfe48 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs @@ -1,11 +1,13 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; namespace Microsoft.Azure.Functions.Worker { - [SupportsDeferredBinding] + [AllowConverterFallback(true)] + [InputConverter(typeof(BlobStorageConverter))] public sealed class BlobTriggerAttribute : TriggerBindingAttribute { private readonly string _blobPath; diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/StorageExtensionStartup.cs b/extensions/Worker.Extensions.Storage.Blobs/src/StorageExtensionStartup.cs index a7bee100d..9f8b10bb1 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/StorageExtensionStartup.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/StorageExtensionStartup.cs @@ -24,11 +24,6 @@ public override void Configure(IFunctionsWorkerApplicationBuilder applicationBui applicationBuilder.Services.AddAzureClientsCore(); // Adds AzureComponentFactory applicationBuilder.Services.AddOptions(); applicationBuilder.Services.AddSingleton, BlobStorageBindingOptionsSetup>(); - - applicationBuilder.Services.Configure((workerOption) => - { - workerOption.InputConverters.RegisterAt(0); - }); } } } diff --git a/test/DotNetWorkerTests/DotNetWorkerTests.csproj b/test/DotNetWorkerTests/DotNetWorkerTests.csproj index 330b5f923..6624a9c84 100644 --- a/test/DotNetWorkerTests/DotNetWorkerTests.csproj +++ b/test/DotNetWorkerTests/DotNetWorkerTests.csproj @@ -24,6 +24,7 @@ + diff --git a/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs b/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs index 2edffc281..34dc65348 100644 --- a/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs +++ b/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs @@ -1,9 +1,14 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading; using Microsoft.Azure.Functions.Tests; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Core; using Microsoft.Azure.Functions.Worker.Grpc.Messages; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.Azure.Functions.Worker.Invocation; @@ -74,6 +79,74 @@ public void Creates() }); } + [Fact] + public void GrpcFunctionDefinition_BlobInput_Creates() + { + using var testVariables = new TestScopedEnvironmentVariable("FUNCTIONS_WORKER_DIRECTORY", "."); + + var bindingInfoProvider = new DefaultOutputBindingsInfoProvider(); + var methodInfoLocator = new DefaultMethodInfoLocator(); + + string fullPathToThisAssembly = GetType().Assembly.Location; + var functionLoadRequest = new FunctionLoadRequest + { + FunctionId = "abc", + Metadata = new RpcFunctionMetadata + { + EntryPoint = $"Microsoft.Azure.Functions.Worker.Tests.{nameof(GrpcFunctionDefinitionTests)}+{nameof(MyBlobFunctionClass)}.{nameof(MyBlobFunctionClass.Run)}", + ScriptFile = Path.GetFileName(fullPathToThisAssembly), + Name = "myfunction" + } + }; + + // We base this on the request exclusively, not the binding attributes. + functionLoadRequest.Metadata.Bindings.Add("req", new BindingInfo { Type = "HttpTrigger", Direction = Direction.In }); + functionLoadRequest.Metadata.Bindings.Add("$return", new BindingInfo { Type = "Http", Direction = Direction.Out }); + + FunctionDefinition definition = functionLoadRequest.ToFunctionDefinition(methodInfoLocator); + + Assert.Equal(functionLoadRequest.FunctionId, definition.Id); + Assert.Equal(functionLoadRequest.Metadata.EntryPoint, definition.EntryPoint); + Assert.Equal(functionLoadRequest.Metadata.Name, definition.Name); + Assert.Equal(fullPathToThisAssembly, definition.PathToAssembly); + + // Parameters + Assert.Collection(definition.Parameters, + p => + { + Assert.Equal("req", p.Name); + Assert.Equal(typeof(HttpRequestData), p.Type); + }, + q => + { + Assert.Equal("myBlob", q.Name); + Assert.Equal(typeof(string), q.Type); + Assert.Contains(PropertyBagKeys.AllowConverterFallback, q.Properties.Keys); + Assert.Contains(PropertyBagKeys.BindingAttributeSupportedConverters, q.Properties.Keys); + Assert.True(true, q.Properties[PropertyBagKeys.AllowConverterFallback].ToString()); + Assert.Contains(new Dictionary>().ToString(), q.Properties[PropertyBagKeys.BindingAttributeSupportedConverters].ToString()); + }); + + // InputBindings + Assert.Collection(definition.InputBindings, + p => + { + Assert.Equal("req", p.Key); + Assert.Equal(BindingDirection.In, p.Value.Direction); + Assert.Equal("HttpTrigger", p.Value.Type); + }); + + // OutputBindings + Assert.Collection(definition.OutputBindings, + p => + { + Assert.Equal("$return", p.Key); + Assert.Equal(BindingDirection.Out, p.Value.Direction); + Assert.Equal("Http", p.Value.Type); + }); + } + + private class MyFunctionClass { public HttpResponseData Run(HttpRequestData req) @@ -89,5 +162,16 @@ public HttpResponseData Run(HttpRequestData req, CancellationToken cancellationT return req.CreateResponse(); } } + + private class MyBlobFunctionClass + { + public HttpResponseData Run( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("input-container/{id}.txt")] string myBlob) + { + return req.CreateResponse(); + } + } + } } diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index f1b1effa2..57e64ae30 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -6,9 +6,11 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Dynamic; +using System.IO; using System.Linq; using System.Reflection; using System.Threading.Tasks; +using Azure.Storage.Blobs; using Microsoft.Azure.Functions.Tests; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; @@ -245,19 +247,19 @@ void ValidateQueueOutput(ExpandoObject b) } [Fact] - public void StorageFunction_SDKTypeBindings() + public void BlobStorageFunctions_SDKTypeBindings() { var generator = new FunctionMetadataGenerator(); var module = ModuleDefinition.ReadModule(_thisAssembly.Location); - var typeDef = TestUtility.GetTypeDefinition(typeof(SDKTypeBindings)); + var typeDef = TestUtility.GetTypeDefinition(typeof(SDKTypeBindings_BlobStorage)); var functions = generator.GenerateFunctionMetadata(typeDef); var extensions = generator.Extensions; - Assert.Equal(1, functions.Count()); + Assert.Equal(5, functions.Count()); - var blobToBlob = functions.Single(p => p.Name == "BlobToBlobFunction"); + var blobStringToBlobStringFunction = functions.Single(p => p.Name == "BlobStringToBlobStringFunction"); - ValidateFunction(blobToBlob, "BlobToBlobFunction", GetEntryPoint(nameof(SDKTypeBindings), nameof(SDKTypeBindings.BlobToBlob)), + ValidateFunction(blobStringToBlobStringFunction, "BlobStringToBlobStringFunction", GetEntryPoint(nameof(SDKTypeBindings_BlobStorage), nameof(SDKTypeBindings_BlobStorage.BlobStringToBlobStringFunction)), b => ValidateBlobTrigger(b), b => ValidateBlobInput(b), b => ValidateBlobOutput(b)); @@ -267,6 +269,38 @@ public void StorageFunction_SDKTypeBindings() { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.0-beta.1" }, }); + var blobClientToBlobStringFunction = functions.Single(p => p.Name == "BlobClientToBlobStringFunction"); + + ValidateFunction(blobClientToBlobStringFunction, "BlobClientToBlobStringFunction", GetEntryPoint(nameof(SDKTypeBindings_BlobStorage), nameof(SDKTypeBindings_BlobStorage.BlobClientToBlobStreamFunction)), + b => ValidateBlobTrigger(b), + b => ValidateBlobInput(b), + b => ValidateBlobOutput(b)); + + + var blobUnsupportedTypeToBlobStringFunction = functions.Single(p => p.Name == "BlobUnsupportedTypeToBlobStringFunction"); + + ValidateFunction(blobUnsupportedTypeToBlobStringFunction, "BlobUnsupportedTypeToBlobStringFunction", GetEntryPoint(nameof(SDKTypeBindings_BlobStorage), nameof(SDKTypeBindings_BlobStorage.BlobUnsupportedTypeToBlobClientFunction)), + b => ValidateBlobTrigger(b), + b => ValidateBlobInput(b), + b => ValidateBlobOutput(b)); + + + var blobPocoToBlobUnsupportedType = functions.Single(p => p.Name == "BlobPocoToBlobUnsupportedType"); + + ValidateFunction(blobPocoToBlobUnsupportedType, "BlobPocoToBlobUnsupportedType", GetEntryPoint(nameof(SDKTypeBindings_BlobStorage), nameof(SDKTypeBindings_BlobStorage.BlobPocoToBlobUnsupportedType)), + b => ValidateBlobTrigger(b), + b => ValidateBlobInput(b), + b => ValidateBlobOutput(b)); + + + var blobByteArrayToBlobByteArrayFunction = functions.Single(p => p.Name == "BlobByteArrayToBlobByteArrayFunction"); + + ValidateFunction(blobByteArrayToBlobByteArrayFunction, "BlobByteArrayToBlobByteArrayFunction", GetEntryPoint(nameof(SDKTypeBindings_BlobStorage), nameof(SDKTypeBindings_BlobStorage.BlobByteArrayToBlobByteArrayFunction)), + b => ValidateBlobTrigger(b), + b => ValidateBlobInput(b), + b => ValidateBlobOutput(b)); + + void ValidateBlobTrigger(ExpandoObject b) { AssertExpandoObject(b, new Dictionary @@ -306,6 +340,105 @@ void ValidateBlobOutput(ExpandoObject b) } } + [Fact] + public void BlobCollectionFunctions_SDKTypeBindings() + { + var generator = new FunctionMetadataGenerator(); + var module = ModuleDefinition.ReadModule(_thisAssembly.Location); + var typeDef = TestUtility.GetTypeDefinition(typeof(SDKTypeBindings_BlobCollection)); + var functions = generator.GenerateFunctionMetadata(typeDef); + var extensions = generator.Extensions; + + Assert.Equal(4, functions.Count()); + + AssertDictionary(extensions, new Dictionary + { + { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.0-beta.1" }, + }); + + var blobStringToBlobClientEnumerable = functions.Single(p => p.Name == "BlobStringToBlobClientEnumerable"); + + ValidateFunction(blobStringToBlobClientEnumerable, "BlobStringToBlobClientEnumerable", GetEntryPoint(nameof(SDKTypeBindings_BlobCollection), nameof(SDKTypeBindings_BlobCollection.BlobStringToBlobClientEnumerable)), + b => ValidateBlobTrigger(b), + b => ValidateBlobInputForEnumerable(b), + b => ValidateBlobOutput(b)); + + void ValidateBlobInputForEnumerable(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "blobinput" }, + { "Type", "blob" }, + { "Direction", "In" }, + { "blobPath", "container2" }, + { "Cardinality", "Many" }, + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding", "True" } } } + }); + } + + var blobStringToBlobStringArray = functions.Single(p => p.Name == "BlobStringToBlobStringArray"); + + ValidateFunction(blobStringToBlobStringArray, "BlobStringToBlobStringArray", GetEntryPoint(nameof(SDKTypeBindings_BlobCollection), nameof(SDKTypeBindings_BlobCollection.BlobStringToBlobStringArray)), + b => ValidateBlobTrigger(b), + b => ValidateBlobInputForStringArray(b), + b => ValidateBlobOutput(b)); + + void ValidateBlobInputForStringArray(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "blobinput" }, + { "Type", "blob" }, + { "Direction", "In" }, + { "blobPath", "container2" }, + { "Cardinality", "Many" }, + { "DataType", "String" }, + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding", "True" } } } + }); + } + + var blobStringToBlobPocoEnumerable = functions.Single(p => p.Name == "BlobStringToBlobPocoEnumerable"); + + ValidateFunction(blobStringToBlobPocoEnumerable, "BlobStringToBlobPocoEnumerable", GetEntryPoint(nameof(SDKTypeBindings_BlobCollection), nameof(SDKTypeBindings_BlobCollection.BlobStringToBlobPocoEnumerable)), + b => ValidateBlobTrigger(b), + b => ValidateBlobInputForEnumerable(b), + b => ValidateBlobOutput(b)); + + + var blobStringToBlobPocoArray = functions.Single(p => p.Name == "BlobStringToBlobPocoArray"); + + ValidateFunction(blobStringToBlobPocoArray, "BlobStringToBlobPocoArray", GetEntryPoint(nameof(SDKTypeBindings_BlobCollection), nameof(SDKTypeBindings_BlobCollection.BlobStringToBlobPocoArray)), + b => ValidateBlobTrigger(b), + b => ValidateBlobInputForEnumerable(b), + b => ValidateBlobOutput(b)); + + + void ValidateBlobTrigger(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "blob" }, + { "Type", "blobTrigger" }, + { "Direction", "In" }, + { "path", "container2/%file%" }, + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } + }); + } + + void ValidateBlobOutput(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "$return" }, + { "Type", "blob" }, + { "Direction", "Out" }, + { "blobPath", "container1/hello.txt" }, + { "Connection", "MyOtherConnection" }, + { "Properties", new Dictionary() } + }); + } + } + [Fact] public void TimerFunction() { @@ -812,16 +945,93 @@ public object BlobToBlobs( } } - private class SDKTypeBindings + private class SDKTypeBindings_BlobStorage { - [Function("BlobToBlobFunction")] + [Function("BlobStringToBlobStringFunction")] [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] - public object BlobToBlob( + public object BlobStringToBlobStringFunction( [BlobTrigger("container2/%file%")] string blob, [BlobInput("container2/%file%")] string blobinput) { throw new NotImplementedException(); } + + + [Function("BlobClientToBlobStringFunction")] + [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] + public object BlobClientToBlobStreamFunction( + [BlobTrigger("container2/%file%")] BlobClient blob, + [BlobInput("container2/%file%")] Stream blobinput) + { + throw new NotImplementedException(); + } + + [Function("BlobByteArrayToBlobByteArrayFunction")] + [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] + public object BlobByteArrayToBlobByteArrayFunction( + [BlobTrigger("container2/%file%")] byte[] blob, + [BlobInput("container2/%file%")] byte[] blobinput) + { + throw new NotImplementedException(); + } + + [Function("BlobUnsupportedTypeToBlobStringFunction")] + [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] + public object BlobUnsupportedTypeToBlobClientFunction( + [BlobTrigger("container2/%file%")] BinaryData blob, + [BlobInput("container2/%file%")] BlobClient blobinput) + { + throw new NotImplementedException(); + } + + [Function("BlobPocoToBlobUnsupportedType")] + [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] + public object BlobPocoToBlobUnsupportedType( + [BlobTrigger("container2/%file%")] Poco blob, + [BlobInput("container2/%file%")] BinaryData blobinput) + { + throw new NotImplementedException(); + } + } + + private class SDKTypeBindings_BlobCollection + { + [Function("BlobStringToBlobStringArray")] + [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] + public object BlobStringToBlobStringArray( + [BlobTrigger("container2/%file%")] string blob, + [BlobInput("container2", IsBatched = true)] string[] blobinput) + { + throw new NotImplementedException(); + } + + + [Function("BlobStringToBlobClientEnumerable")] + [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] + public object BlobStringToBlobClientEnumerable( + [BlobTrigger("container2/%file%")] string blob, + [BlobInput("container2", IsBatched = true)] IEnumerable blobinput) + { + throw new NotImplementedException(); + } + + [Function("BlobStringToBlobPocoEnumerable")] + [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] + public object BlobStringToBlobPocoEnumerable( + [BlobTrigger("container2/%file%")] string blob, + [BlobInput("container2", IsBatched = true)] IEnumerable blobinput) + { + throw new NotImplementedException(); + } + + [Function("BlobStringToBlobPocoArray")] + [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] + public object BlobStringToBlobPocoArray( + [BlobTrigger("container2/%file%")] string blob, + [BlobInput("container2", IsBatched = true)] Poco[] blobinput) + { + throw new NotImplementedException(); + } } private class ExternalType_Return diff --git a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs index b19c23437..bf2b86883 100644 --- a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs +++ b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs @@ -526,43 +526,6 @@ await TestHelpers.RunTestAsync( expectedGeneratedFileName, expectedOutput); } - - [Fact (Skip = "Depends on source gen description cleanup, issue #1323")] - public async void MultipleOutputOnMethodFails() - { - var inputCode = @"using System; - using System.Net; - using System.Collections; - using System.Collections.Generic; - using Microsoft.Azure.Functions.Worker; - using Microsoft.Azure.Functions.Worker.Http; - using System.Linq; - using System.Threading.Tasks; - - namespace FunctionApp - { - public class EventHubsInput - { - [Function(""QueueToBlobFunction"")] - [BlobOutput(""container1/hello.txt"", Connection = ""MyOtherConnection"")] - [QueueOutput(""queue2"")] - public string QueueToBlob( - [QueueTrigger(""queueName"", Connection = ""MyConnection"")] string queuePayload) - { - throw new NotImplementedException(); - } - } - }"; - - string? expectedGeneratedFileName = null; - string? expectedOutput = null; - - await TestHelpers.RunTestAsync( - _referencedExtensionAssemblies, - inputCode, - expectedGeneratedFileName, - expectedOutput); - } } } } diff --git a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata new file mode 100644 index 000000000..2d151241b --- /dev/null +++ b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata @@ -0,0 +1,792 @@ +[ + { + "name": "BlobInputClientFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputClientFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "client", + "direction": "In", + "type": "blob", + "blobPath": "input-container/sample1.txt", + "cardinality": "One", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "BlobInputStreamFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputStreamFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "stream", + "direction": "In", + "type": "blob", + "blobPath": "input-container/sample1.txt", + "cardinality": "One", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "BlobInputByteArrayFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputByteArrayFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "data", + "direction": "In", + "type": "blob", + "blobPath": "input-container/sample1.txt", + "cardinality": "One", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "BlobInputStringFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputStringFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "data", + "direction": "In", + "type": "blob", + "blobPath": "input-container/{filename}", + "cardinality": "One", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "BlobInputBookFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputBookFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "data", + "direction": "In", + "type": "blob", + "blobPath": "input-container/book.json", + "cardinality": "One", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "BlobInputCollectionFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputCollectionFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "blobs", + "direction": "In", + "type": "blob", + "blobPath": "input-container", + "cardinality": "Many", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "BlobInputStringArrayFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputStringArrayFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "blobContent", + "direction": "In", + "type": "blob", + "blobPath": "input-container", + "cardinality": "Many", + "dataType": "String", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "BlobInputBookArrayFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputBookArrayFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "books", + "direction": "In", + "type": "blob", + "blobPath": "input-container", + "cardinality": "Many", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "BlobClientFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobTriggerBindingSamples.BlobClientFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "client", + "direction": "In", + "type": "blobTrigger", + "path": "client-trigger/{name}", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "BlobStreamFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobTriggerBindingSamples.BlobStreamFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "stream", + "direction": "In", + "type": "blobTrigger", + "path": "stream-trigger/{name}", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "BlobByteArrayFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobTriggerBindingSamples.BlobByteArrayFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "data", + "direction": "In", + "type": "blobTrigger", + "path": "byte-trigger", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "BlobStringFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobTriggerBindingSamples.BlobStringFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "data", + "direction": "In", + "type": "blobTrigger", + "path": "string-trigger", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "BlobBookFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.BlobTriggerBindingSamples.BlobBookFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "data", + "direction": "In", + "type": "blobTrigger", + "path": "book-trigger", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "ExpressionFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.ExpressionFunction.Run", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "book", + "direction": "In", + "type": "queueTrigger", + "queueName": "expression-trigger", + "properties": {} + }, + { + "name": "myBlob", + "direction": "In", + "type": "blob", + "blobPath": "input-container/{id}.txt", + "cardinality": "One", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "DocsByUsingCosmosClient", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsByUsingCosmosClient", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "client", + "direction": "In", + "type": "cosmosDB", + "databaseName": "", + "containerName": "", + "connection": "CosmosDBConnection", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocsByUsingDatabaseClient", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsByUsingDatabaseClient", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "database", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "", + "connection": "CosmosDBConnection", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocsByUsingContainerClient", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsByUsingContainerClient", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "container", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromQueryString", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromQueryString", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "toDoItem", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "id": "{Query.id}", + "partitionKey": "{Query.partitionKey}", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromRouteData", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromRouteData", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "route": "todoitems/{partitionKey}/{id}", + "properties": {} + }, + { + "name": "toDoItem", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "id": "{id}", + "partitionKey": "{partitionKey}", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromRouteDataUsingSqlQuery", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromRouteDataUsingSqlQuery", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "route": "todoitems2/{id}", + "properties": {} + }, + { + "name": "toDoItems", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "sqlQuery": "SELECT * FROM ToDoItems t where t.id = {id}", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromQueryStringUsingSqlQuery", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromQueryStringUsingSqlQuery", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "toDoItems", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "sqlQuery": "SELECT * FROM ToDoItems t where t.id = {id}", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocsBySqlQuery", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsBySqlQuery", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "toDoItems", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "sqlQuery": "SELECT * FROM ToDoItems t WHERE CONTAINS(t.description, 'cat')", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromJSON", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromJSON", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "toDoItemLookup", + "direction": "In", + "type": "queueTrigger", + "queueName": "todoqueueforlookup", + "properties": {} + }, + { + "name": "toDoItem", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "id": "{ToDoItemId}", + "partitionKey": "{ToDoItemPartitionKeyValue}", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "CosmosTriggerFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosTriggerFunction.Run", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "todoItems", + "direction": "In", + "type": "cosmosDBTrigger", + "databaseName": "ToDoItems", + "containerName": "TriggerItems", + "connection": "CosmosDBConnection", + "createLeaseContainerIfNotExists": true, + "properties": {} + } + ] + } +] \ No newline at end of file diff --git a/test/SdkE2ETests/PublishTests.cs b/test/SdkE2ETests/PublishTests.cs index 0e5d31297..a69130b61 100644 --- a/test/SdkE2ETests/PublishTests.cs +++ b/test/SdkE2ETests/PublishTests.cs @@ -74,7 +74,66 @@ private async Task RunPublishTest(string outputDir, string additionalParams = nu Assert.True(JToken.DeepEquals(extensionsJsonContents, expected), $"Actual: {extensionsJsonContents}{Environment.NewLine}Expected: {expected}"); // Verify functions.metadata - TestUtility.ValidateFunctionsMetadata(functionsMetadataPath, "functions.metadata"); + TestUtility.ValidateFunctionsMetadata(functionsMetadataPath, "Microsoft.Azure.Functions.SdkE2ETests.Contents.functions.metadata"); + } + + + [Fact] + public async Task Publish_SdkTypeBindings() + { + string outputDir = await TestUtility.InitializeTestAsync(_testOutputHelper, nameof(Publish_SdkTypeBindings)); + await RunPublishTestForSdkTypeBindings(outputDir); + } + + [Fact] + public async Task Publish_Rid_SdkTypeBindings() + { + string outputDir = await TestUtility.InitializeTestAsync(_testOutputHelper, nameof(Publish_Rid_SdkTypeBindings)); + await RunPublishTestForSdkTypeBindings(outputDir, "-r win-x86"); + } + + private async Task RunPublishTestForSdkTypeBindings(string outputDir, string additionalParams = null) + { + // Name of the csproj + string projectFileDirectory = Path.Combine(TestUtility.SamplesRoot, "WorkerBindingSamples", "WorkerBindingSamples.csproj"); + + await TestUtility.RestoreAndPublishProjectAsync(projectFileDirectory, outputDir, additionalParams, _testOutputHelper); + + // Make sure files are in /.azurefunctions + string azureFunctionsDir = Path.Combine(outputDir, ".azurefunctions"); + Assert.True(Directory.Exists(azureFunctionsDir)); + var files = Directory.EnumerateFiles(azureFunctionsDir); + + // Verify files are present + string metadataLoaderPath = Path.Combine(azureFunctionsDir, "Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.dll"); + string extensionsJsonPath = Path.Combine(outputDir, "extensions.json"); + string functionsMetadataPath = Path.Combine(outputDir, "functions.metadata"); + Assert.True(File.Exists(metadataLoaderPath)); + Assert.True(File.Exists(extensionsJsonPath)); + Assert.True(File.Exists(functionsMetadataPath)); + + // Verify extensions.json + JObject jObjects = JObject.Parse(File.ReadAllText(extensionsJsonPath)); + JObject extensionsJsonContents = jObjects; + JToken expected = JObject.FromObject(new + { + extensions = new[] + { + new Extension("Startup", + "Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.Startup, Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader, Version=1.0.0.0, Culture=neutral, PublicKeyToken=551316b6919f366c", + @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.dll"), + new Extension("AzureStorageBlobs", + "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageBlobsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Blobs, Version=5.1.0.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", + @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.dll"), + new Extension("AzureStorageQueues", + "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageQueuesWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Queues, Version=5.0.0.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", + @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Queues.dll") + } + }); + Assert.True(JToken.DeepEquals(extensionsJsonContents, expected), $"Actual: {extensionsJsonContents}{Environment.NewLine}Expected: {expected}"); + + // Verify functions.metadata + TestUtility.ValidateFunctionsMetadata(functionsMetadataPath, "Microsoft.Azure.Functions.SdkE2ETests.Contents.WorkerBindingSamplesOutput.functions.metadata"); } private class Extension diff --git a/test/SdkE2ETests/SdkE2ETests.csproj b/test/SdkE2ETests/SdkE2ETests.csproj index 8e4296318..37d319059 100644 --- a/test/SdkE2ETests/SdkE2ETests.csproj +++ b/test/SdkE2ETests/SdkE2ETests.csproj @@ -10,10 +10,12 @@ + + From 6513268317ea67e1d0783c2e625590596dc094a9 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Mon, 5 Jun 2023 10:17:46 -0700 Subject: [PATCH 17/47] Add new blob E2E test and enable collection tests (#1591) --- .../E2EApp/Blob/BlobInputBindingFunctions.cs | 54 +++++++++++++++++-- .../E2ETests/Storage/BlobEndToEndTests.cs | 20 ++++--- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs b/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs index 4e6798c90..3452b7147 100644 --- a/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs +++ b/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs @@ -8,6 +8,7 @@ using System.Text; using System.Threading.Tasks; using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.Extensions.Logging; @@ -33,6 +34,50 @@ public async Task BlobInputClientTest( return response; } + [Function(nameof(BlobInputBlockClientTest))] + public async Task BlobInputBlockClientTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] BlockBlobClient client) + { + var downloadResult = await client.DownloadContentAsync(); + var response = req.CreateResponse(HttpStatusCode.OK); + await response.Body.WriteAsync(downloadResult.Value.Content); + return response; + } + + [Function(nameof(BlobInputAppendClientTest))] + public async Task BlobInputAppendClientTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] AppendBlobClient client) + { + var downloadResult = await client.DownloadContentAsync(); + var response = req.CreateResponse(HttpStatusCode.OK); + await response.Body.WriteAsync(downloadResult.Value.Content); + return response; + } + + [Function(nameof(BlobInputPageClientTest))] + public async Task BlobInputPageClientTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] PageBlobClient client) + { + var downloadResult = await client.DownloadContentAsync(); + var response = req.CreateResponse(HttpStatusCode.OK); + await response.Body.WriteAsync(downloadResult.Value.Content); + return response; + } + + [Function(nameof(BlobInputBaseClientTest))] + public async Task BlobInputBaseClientTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] BlobBaseClient client) + { + var downloadResult = await client.DownloadContentAsync(); + var response = req.CreateResponse(HttpStatusCode.OK); + await response.Body.WriteAsync(downloadResult.Value.Content); + return response; + } + [Function(nameof(BlobInputContainerClientTest))] public async Task BlobInputContainerClientTest( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, @@ -100,7 +145,8 @@ public async Task BlobInputCollectionTest( } var response = req.CreateResponse(HttpStatusCode.OK); - await response.WriteStringAsync(blobList.ToString()); + string contentAsString = string.Join(", ", blobList.ToArray()); + await response.WriteStringAsync(contentAsString); return response; } @@ -110,7 +156,8 @@ public async Task BlobInputStringArrayTest( [BlobInput("test-input-dotnet-isolated", IsBatched = true)] string[] blobContent) { var response = req.CreateResponse(HttpStatusCode.OK); - await response.WriteStringAsync(blobContent.ToString()); + string contentAsString = string.Join(", ", blobContent); + await response.WriteStringAsync(contentAsString); return response; } @@ -127,7 +174,8 @@ public async Task BlobInputPocoArrayTest( } var response = req.CreateResponse(HttpStatusCode.OK); - await response.WriteStringAsync(bookNames.ToString()); + string contentAsString = string.Join(", ", bookNames.ToArray()); + await response.WriteStringAsync(contentAsString); return response; } } diff --git a/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs b/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs index a3f41d63c..708c7d04c 100644 --- a/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs +++ b/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs @@ -163,6 +163,10 @@ await TestUtility.RetryAsync(() => [Theory] [InlineData("BlobInputClientTest")] + [InlineData("BlobInputBlockClientTest")] + [InlineData("BlobInputAppendClientTest")] + [InlineData("BlobInputPageClientTest")] + [InlineData("BlobInputBaseClientTest")] [InlineData("BlobInputContainerClientTest")] [InlineData("BlobInputStreamTest")] [InlineData("BlobInputByteTest")] @@ -210,10 +214,10 @@ public async Task BlobInput_Poco_Succeeds() Assert.Contains(expectedMessage, actualMessage); } - [Fact(Skip = "Collection support released in host version 4.16+")] + [Fact] public async Task BlobInput_BlobClientCollection_Succeeds() { - string expectedMessage = "testFile1, testFile2, testFile3"; + string expectedMessage = "testFile1.txt, testFile2.txt, testFile3.txt"; HttpStatusCode expectedStatusCode = HttpStatusCode.OK; //Cleanup @@ -230,10 +234,10 @@ public async Task BlobInput_BlobClientCollection_Succeeds() //Verify Assert.Equal(expectedStatusCode, response.StatusCode); - Assert.Contains(expectedMessage, actualMessage); + Assert.Equal(expectedMessage, actualMessage); } - [Fact(Skip = "Collection support released in host version 4.16+")] + [Fact] public async Task BlobInput_StringCollection_Succeeds() { string expectedMessage = "ABC, DEF, GHI"; @@ -248,15 +252,15 @@ public async Task BlobInput_StringCollection_Succeeds() await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile3", "GHI"); //Trigger - HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputCollectionTest"); + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputStringArrayTest"); string actualMessage = await response.Content.ReadAsStringAsync(); //Verify Assert.Equal(expectedStatusCode, response.StatusCode); - Assert.Contains(expectedMessage, actualMessage); + Assert.Equal(expectedMessage, actualMessage); } - [Fact(Skip = "Collection support released in host version 4.16+")] + [Fact] public async Task BlobInput_PocoCollection_Succeeds() { string book1 = $@"{{ ""id"": ""1"", ""name"": ""To Kill a Mockingbird""}}"; @@ -281,7 +285,7 @@ public async Task BlobInput_PocoCollection_Succeeds() //Verify Assert.Equal(expectedStatusCode, response.StatusCode); - Assert.Contains(expectedMessage, actualMessage); + Assert.Equal(expectedMessage, actualMessage); } public void Dispose() From 1341beb6ef71972e714b3b3f696cfadc46c4e2f1 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Mon, 17 Jul 2023 16:00:19 -0700 Subject: [PATCH 18/47] Refactor blob extension unit tests (#1578) --- .../src/BlobStorageConverter.cs | 48 +- .../src/Config/BlobStorageBindingOptions.cs | 9 +- .../Blob/BlobStorageConverterTests.cs | 861 +++++++++++++----- 3 files changed, 661 insertions(+), 257 deletions(-) diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs index b0c4be9f0..30a5a8ce9 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -46,7 +46,7 @@ public async ValueTask ConvertAsync(ConverterContext context) }; } - internal virtual async ValueTask ConvertFromBindingDataAsync(ConverterContext context, ModelBindingData modelBindingData) + private async ValueTask ConvertFromBindingDataAsync(ConverterContext context, ModelBindingData modelBindingData) { if (!IsBlobExtension(modelBindingData)) { @@ -71,7 +71,7 @@ internal virtual async ValueTask ConvertFromBindingDataAsync(C return ConversionResult.Unhandled(); } - internal virtual async ValueTask ConvertFromCollectionBindingDataAsync(ConverterContext context, CollectionModelBindingData collectionModelBindingData) + private async ValueTask ConvertFromCollectionBindingDataAsync(ConverterContext context, CollectionModelBindingData collectionModelBindingData) { var blobCollection = new List(collectionModelBindingData.ModelBindingDataArray.Length); Type elementType = context.TargetType.IsArray ? context.TargetType.GetElementType() : context.TargetType.GenericTypeArguments[0]; @@ -105,7 +105,7 @@ internal virtual async ValueTask ConvertFromCollectionBindingD } } - internal bool IsBlobExtension(ModelBindingData bindingData) + private bool IsBlobExtension(ModelBindingData bindingData) { if (bindingData?.Source is not Constants.BlobExtensionName) { @@ -116,30 +116,35 @@ internal bool IsBlobExtension(ModelBindingData bindingData) return true; } - internal Dictionary GetBindingDataContent(ModelBindingData bindingData) + private Dictionary GetBindingDataContent(ModelBindingData bindingData) { return bindingData?.ContentType switch { Constants.JsonContentType => new Dictionary(bindingData?.Content?.ToObjectFromJson>(), StringComparer.OrdinalIgnoreCase), - _ => throw new NotSupportedException($"Unexpected content-type. Currently only {Constants.JsonContentType} is supported.") + _ => throw new NotSupportedException($"Unexpected content-type. Currently only '{Constants.JsonContentType}' is supported.") }; } - internal virtual async Task ConvertModelBindingDataAsync(IDictionary content, Type targetType, ModelBindingData bindingData) + private async Task ConvertModelBindingDataAsync(IDictionary content, Type targetType, ModelBindingData bindingData) { content.TryGetValue(Constants.Connection, out var connectionName); content.TryGetValue(Constants.ContainerName, out var containerName); content.TryGetValue(Constants.BlobName, out var blobName); - if (string.IsNullOrEmpty(connectionName) || string.IsNullOrEmpty(containerName)) + if (string.IsNullOrEmpty(connectionName)) { - throw new ArgumentNullException("'Connection' and 'ContainerName' cannot be null or empty"); + throw new ArgumentNullException(nameof(connectionName)); + } + + if (string.IsNullOrEmpty(containerName)) + { + throw new ArgumentNullException(nameof(containerName)); } return await ToTargetTypeAsync(targetType, connectionName, containerName, blobName); } - internal virtual async Task ToTargetTypeAsync(Type targetType, string connectionName, string containerName, string blobName) => targetType switch + private async Task ToTargetTypeAsync(Type targetType, string connectionName, string containerName, string blobName) => targetType switch { Type _ when targetType == typeof(String) => await GetBlobStringAsync(connectionName, containerName, blobName), Type _ when targetType == typeof(Stream) => await GetBlobStreamAsync(connectionName, containerName, blobName), @@ -153,13 +158,13 @@ internal Dictionary GetBindingDataContent(ModelBindingData bindi _ => await DeserializeToTargetObjectAsync(targetType, connectionName, containerName, blobName) }; - internal async Task DeserializeToTargetObjectAsync(Type targetType, string connectionName, string containerName, string blobName) + private async Task DeserializeToTargetObjectAsync(Type targetType, string connectionName, string containerName, string blobName) { var content = await GetBlobStreamAsync(connectionName, containerName, blobName); return _workerOptions?.Value?.Serializer?.Deserialize(content, targetType, CancellationToken.None); } - internal object? ToTargetTypeCollection(IEnumerable blobCollection, string methodName, Type type) + private object? ToTargetTypeCollection(IEnumerable blobCollection, string methodName, Type type) { blobCollection = blobCollection.Select(b => Convert.ChangeType(b, type)); MethodInfo method = typeof(BlobStorageConverter).GetMethod(methodName, BindingFlags.Static | BindingFlags.NonPublic); @@ -168,44 +173,39 @@ internal Dictionary GetBindingDataContent(ModelBindingData bindi return genericMethod.Invoke(null, new[] { blobCollection.ToList() }); } - internal static T[] CloneToArray(IList source) + private static T[] CloneToArray(IList source) { return source.Cast().ToArray(); } - internal static IEnumerable CloneToList(IList source) + private static IEnumerable CloneToList(IList source) { return source.Cast(); } - internal virtual async Task GetBlobStringAsync(string connectionName, string containerName, string blobName) + private async Task GetBlobStringAsync(string connectionName, string containerName, string blobName) { var client = CreateBlobClient(connectionName, containerName, blobName); - return await GetBlobContentStringAsync(client); - } - - private async Task GetBlobContentStringAsync(BlobClient client) - { var download = await client.DownloadContentAsync(); return download.Value.Content.ToString(); } - internal virtual async Task GetBlobBinaryDataAsync(string connectionName, string containerName, string blobName) + private async Task GetBlobBinaryDataAsync(string connectionName, string containerName, string blobName) { using MemoryStream stream = new(); var client = CreateBlobClient(connectionName, containerName, blobName); - await client.DownloadToAsync(stream); + var res = await client.DownloadToAsync(stream); return stream.ToArray(); } - internal virtual async Task GetBlobStreamAsync(string connectionName, string containerName, string blobName) + private async Task GetBlobStreamAsync(string connectionName, string containerName, string blobName) { var client = CreateBlobClient(connectionName, containerName, blobName); var download = await client.DownloadStreamingAsync(); return download.Value.Content; } - internal virtual BlobContainerClient CreateBlobContainerClient(string connectionName, string containerName) + private BlobContainerClient CreateBlobContainerClient(string connectionName, string containerName) { var blobStorageOptions = _blobOptions.Get(connectionName); BlobServiceClient blobServiceClient = blobStorageOptions.CreateClient(); @@ -213,7 +213,7 @@ internal virtual BlobContainerClient CreateBlobContainerClient(string connection return container; } - internal virtual T CreateBlobClient(string connectionName, string containerName, string blobName) where T : BlobBaseClient + private T CreateBlobClient(string connectionName, string containerName, string blobName) where T : BlobBaseClient { if (string.IsNullOrEmpty(blobName)) { diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptions.cs b/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptions.cs index 7c82528e6..92662bbd4 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptions.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptions.cs @@ -17,8 +17,15 @@ internal class BlobStorageBindingOptions public BlobClientOptions? BlobClientOptions { get; set; } - public BlobServiceClient CreateClient() + internal BlobServiceClient? Client { get; set; } + + internal BlobServiceClient CreateClient() { + if (this.Client is not null) + { + return this.Client; + } + if (ServiceUri is not null && Credential is not null) { return new BlobServiceClient(ServiceUri, Credential, BlobClientOptions); diff --git a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs index 903d7a7f1..d48a0f618 100644 --- a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs +++ b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs @@ -6,14 +6,16 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.Json; +using System.Threading; using System.Threading.Tasks; +using Azure; using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Specialized; +using Azure.Storage.Blobs.Models; using Google.Protobuf; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Grpc.Messages; -using Microsoft.Azure.Functions.Worker.Tests; using Microsoft.Azure.Functions.Worker.Tests.Converters; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -22,375 +24,771 @@ using Moq; using Xunit; -using Constants = Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs.Constants; - +// Scenarios for BlobBaseClient, BlockBlobClient, PageBlobClient, and AppendBlobClient +// are tested via E2E tests as Moq does not support mocking extension methods directly namespace Microsoft.Azure.Functions.WorkerExtension.Tests { public class BlobStorageConverterTests { - private Mock _mockBlobStorageConverter; + private BlobStorageConverter _blobStorageConverter; + private Mock _mockBlobServiceClient; public BlobStorageConverterTests() { var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + var workerOptions = host.Services.GetService>(); - var blobOptions = host.Services.GetService>(); var logger = host.Services.GetService>(); - _mockBlobStorageConverter = new Mock(workerOptions, blobOptions, logger); - _mockBlobStorageConverter.CallBase = true; + _mockBlobServiceClient = new Mock(); + + var mockBlobOptions = new Mock(); + mockBlobOptions.Object.Client = _mockBlobServiceClient.Object; + + var mockBlobOptionsSnapshot = new Mock>(); + mockBlobOptionsSnapshot + .Setup(m => m.Get(It.IsAny())) + .Returns(mockBlobOptions.Object); + + _blobStorageConverter = new BlobStorageConverter(workerOptions, mockBlobOptionsSnapshot.Object, logger); } [Fact] - public async Task ConvertAsync_SourceAsObject_ReturnsUnhandled() + public async Task ConvertAsync_ValidModelBindingData_BlobClient_ReturnsSuccess() { - var context = new TestConverterContext(typeof(string), new Object()); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - var conversionResult = await _mockBlobStorageConverter.Object.ConvertAsync(context); + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("MyBlob", clientResult.Name); } - [Fact] - public async Task ConvertAsync_SourceAsModelBindingData_ReturnsSuccess() + [Fact (Skip = "Fails: ChangeType doesn't work for mock objects 'Object must implement IConvertible'")] + public async Task ConvertAsync_ValidModelBindingData_BlobClientCollection_List_ReturnsSuccess() { - object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - var context = new TestConverterContext(typeof(string), source); - _mockBlobStorageConverter - .Setup(c => c.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName)) - .ReturnsAsync("test"); + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (IEnumerable)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(clientResult); + Assert.Equal("MyBlob", clientResult.First().Name); + } + + [Fact (Skip = "Fails: ChangeType doesn't work for mock objects 'Object must implement IConvertible'")] + public async Task ConvertAsync_ValidModelBindingData_BlobClientCollection_Array_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(BlobClient[]), grpcModelBindingData); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - var conversionResult = await _mockBlobStorageConverter.Object.ConvertAsync(context); + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient[])conversionResult.Value; + // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(clientResult); + Assert.Equal("MyBlob", clientResult.First().Name); } [Fact] - public async Task ConvertAsync_SourceAsCollectionModelBindingData_ReturnsSuccess() + public async Task ConvertAsync_ValidModelBindingData_BlobContainerClient_ReturnsSuccess() { - object source = GetTestGrpcCollectionModelBindingData(); - var context = new TestConverterContext(typeof(string), source); - _mockBlobStorageConverter - .Setup(c => c.ConvertFromCollectionBindingDataAsync(context, (Worker.Core.CollectionModelBindingData) source)) - .Returns(new ValueTask(ConversionResult.Success("test"))); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobContainerClient), grpcModelBindingData); - var conversionResult = await _mockBlobStorageConverter.Object.ConvertAsync(context); + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Name).Returns("MyContainer"); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobContainerClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("MyContainer", clientResult.Name); + } + + [Fact (Skip = "Fails: ChangeType doesn't work for mock objects 'Object must implement IConvertible'")] + public async Task ConvertAsync_ValidModelBindingData_BlobContainerClientCollection_List_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Name).Returns("MyContainer"); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (IEnumerable)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(clientResult); + Assert.Equal("MyContainer", clientResult.First().Name); + } + + [Fact (Skip = "Fails: ChangeType doesn't work for mock objects 'Object must implement IConvertible'")] + public async Task ConvertAsync_ValidModelBindingData_BlobContainerClientCollection_Array_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(BlobContainerClient[]), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Name).Returns("MyContainer"); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobContainerClient[])conversionResult.Value; + + // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(clientResult); + Assert.Equal("MyContainer", clientResult.First().Name); } [Fact] - public async Task ConvertAsync_SourceAsModelBindingData_ReturnsFailure() + public async Task ConvertAsync_ValidModelBindingData_String_ReturnsSuccess() { - object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - var context = new TestConverterContext(typeof(string), source); - _mockBlobStorageConverter - .Setup(c => c.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName)) - .ThrowsAsync(new Exception()); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(String), grpcModelBindingData); - var conversionResult = await _mockBlobStorageConverter.Object.ConvertAsync(context); + var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var stringResult = (String)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("MyBlobString", stringResult); } [Fact] - public async Task ConvertFromBindingDataAsync_ReturnsSuccess() + public async Task ConvertAsync_ValidModelBindingData_StringCollection_List_ReturnsSuccess() { - object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - var contentDict = GetTestContentDict(); - var context = new TestConverterContext(typeof(string), source); - var modelBindingData = (Worker.Core.ModelBindingData) source; + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - _mockBlobStorageConverter - .Setup(c => c.ConvertModelBindingDataAsync(contentDict, typeof(string), modelBindingData)) - .ReturnsAsync(new ValueTask(ConversionResult.Success("test"))); + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); - var conversionResult = await _mockBlobStorageConverter.Object.ConvertFromBindingDataAsync(context, modelBindingData); + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var stringResult = (IEnumerable)conversionResult.Value; + + // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(stringResult); + Assert.Equal("MyBlobString", stringResult.First().ToString()); } [Fact] - public async Task ConvertFromBindingDataAsync_ReturnsFailure() + public async Task ConvertAsync_ValidModelBindingData_StringCollection_Array_ReturnsSuccess() { - object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - var contentDict = GetTestContentDict(); - var context = new TestConverterContext(typeof(string), source); + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(String[]), grpcModelBindingData); - _mockBlobStorageConverter - .Setup(c => c.ConvertModelBindingDataAsync(contentDict, typeof(string), (Worker.Core.ModelBindingData)source)) - .ThrowsAsync(new Exception()); + var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - var conversionResult = await _mockBlobStorageConverter.Object.ConvertFromBindingDataAsync(context, (Worker.Core.ModelBindingData)source); + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var stringResult = (String[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(stringResult); + Assert.Equal("MyBlobString", stringResult.First().ToString()); } [Fact] - public async Task ConvertFromBindingDataAsync_ReturnsUnhandled() + public async Task ConvertAsync_ValidModelBindingData_Stream_ReturnsSuccess() { - object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - var context = new TestConverterContext(typeof(string), source); - var dict = _mockBlobStorageConverter.Object.GetBindingDataContent((Worker.Core.ModelBindingData)source); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Stream), grpcModelBindingData); - _mockBlobStorageConverter.Setup(c => c.ConvertModelBindingDataAsync(dict, typeof(string), (Worker.Core.ModelBindingData)source)).ReturnsAsync(null); + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - var conversionResult = await _mockBlobStorageConverter.Object.ConvertFromBindingDataAsync(context, (Worker.Core.ModelBindingData)source); + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var streamResult = (Stream)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedStream, streamResult); } - [Theory] - [InlineData(Constants.BlobExtensionName, true)] - [InlineData(" ", false)] - [InlineData("incorrect-value", false)] - public void IsBlobExtension_MatchesExpectedOutput(string sourceVal, bool expectedResult) + [Fact (Skip = "Fails - broken scenario. ChangeType fails trying to convert MemoryStream to Stream) - 'Object must implement IConvertible'")] + public async Task ConvertAsync_ValidModelBindingData_StreamCollection_List_ReturnsSuccess() { - var grpcModelBindingData = new GrpcModelBindingData(new ModelBindingData() - { - Version = "1.0", - Source = sourceVal, - Content = ByteString.CopyFrom(GetFullTestBinaryData()), - ContentType = "application/json" - }); + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - var result = _mockBlobStorageConverter.Object.IsBlobExtension(grpcModelBindingData); + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var streamResult = (IEnumerable)conversionResult.Value; - Assert.Equal(result, expectedResult); + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(streamResult); + Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); + } + + [Fact (Skip = "Fails - broken scenario. ChangeType fails trying to convert MemoryStream to Stream) - 'Object must implement IConvertible'")] + public async Task ConvertAsync_ValidModelBindingData_StreamCollection_Array_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(Stream[]), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var streamResult = (Stream[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(streamResult); + Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); + Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); } [Fact] - public void GetBindingDataContent_CompleteGrpcModelBindingData_Works() + public async Task ConvertAsync_ValidModelBindingData_ByteArray_ReturnsSuccess() { - var grpcModelBindingData = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - var result = _mockBlobStorageConverter.Object.GetBindingDataContent(grpcModelBindingData); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Byte[]), grpcModelBindingData); + + var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); + var testStream = new MemoryStream(expectedByteArray); + testStream.Position = 0; + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadToAsync(It.IsAny(), default)) + .Callback(async (s, _) => await testStream.CopyToAsync(s)) + .ReturnsAsync(new Mock().Object); - result.TryGetValue(Constants.Connection, out var connectionName); - result.TryGetValue(Constants.ContainerName, out var containerName); - result.TryGetValue(Constants.BlobName, out var blobName); + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - Assert.Equal(3, result.Count); - Assert.Equal(Constants.Connection, connectionName); - Assert.Equal(Constants.ContainerName, containerName); - Assert.Equal(Constants.BlobName, blobName); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var byteResult = (Byte[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(byteResult); + Assert.Equal(new byte[0], byteResult); + // Assert.Equal(expectedByteArray, byteResult); // Running into issues mocking DownloadToAsync stream update } [Fact] - public void GetBindingDataContent_IncompleteGrpcModelBindingData_ReturnsNull() + public async Task ConvertAsync_ValidModelBindingData_ByteArrayCollection_List_ReturnsSuccess() { - var grpcModelBindingData = GetTestGrpcModelBindingData(GetTestBinaryData()); + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); - var result = _mockBlobStorageConverter.Object.GetBindingDataContent(grpcModelBindingData); + var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); + var testStream = new MemoryStream(expectedByteArray); + testStream.Position = 0; - result.TryGetValue(Constants.Connection, out var connectionName); - result.TryGetValue(Constants.ContainerName, out var containerName); - result.TryGetValue(Constants.BlobName, out var blobName); + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadToAsync(It.IsAny(), default)) + .Callback(async (s, _) => await testStream.CopyToAsync(s)) + .ReturnsAsync(new Mock().Object); - Assert.Single(result); - Assert.True(connectionName is null); - Assert.True(containerName is null); - Assert.Equal(Constants.BlobName, blobName); + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var byteResult = (IEnumerable)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(byteResult); + // Assert.Equal(expectedByteArray, byteResult.First()); // Running into issues mocking DownloadToAsync stream update } [Fact] - public void GetBindingDataContent_UnSupportedContentType_Throws() + public async Task ConvertAsync_ValidModelBindingData_ByteArrayCollection_Array_ReturnsSuccess() { - var grpcModelBindingData = new GrpcModelBindingData(new ModelBindingData() - { - Version = "1.0", - Source = Constants.BlobExtensionName, - Content = ByteString.CopyFrom(GetFullTestBinaryData()), - ContentType = "NotSupported" - }); + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(Byte[][]), grpcModelBindingData); - try - { - var dict = _mockBlobStorageConverter.Object.GetBindingDataContent(grpcModelBindingData); - Assert.Fail("Test fails as the expected exception not thrown"); - } - catch (Exception ex) - { - Assert.Equal(typeof(NotSupportedException), ex.GetType()); - } + var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); + var testStream = new MemoryStream(expectedByteArray); + testStream.Position = 0; + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadToAsync(It.IsAny(), default)) + .Callback(async (s, _) => await testStream.CopyToAsync(s)) + .ReturnsAsync(new Mock().Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var byteResult = (Byte[][])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(byteResult); + // Assert.Equal(expectedByteArray, byteResult.First()); // Running into issues mocking DownloadToAsync steam update } [Fact] - public async Task ConvertModelBindingDataAsync_IncompleteGrpcModelBindingData_Throws() + public async Task ConvertAsync_ValidModelBindingData_POCO_ReturnsSuccess() { - var grpcModelBindingData = GetTestGrpcModelBindingData(GetTestBinaryData()); - var dict = _mockBlobStorageConverter.Object.GetBindingDataContent(grpcModelBindingData); - _mockBlobStorageConverter.Setup(c => c.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync("test"); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book), grpcModelBindingData); - try - { - var result = await _mockBlobStorageConverter.Object.ConvertModelBindingDataAsync(dict, typeof(string), grpcModelBindingData); - Assert.Fail("Test fails as the expected exception not thrown"); - } - catch (ArgumentNullException) { } + var expectedBook = new Book() { Name = "MyBook" }; + + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (Book)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedBook.GetType(), pocoResult.GetType()); + Assert.Equal(expectedBook.Name, pocoResult.Name); } [Fact] - public async Task ConvertModelBindingDataAsync_GrpcModelBindingData_Works() + public async Task ConvertAsync_ValidModelBindingData_POCOCollection_List_ReturnsSuccess() { - var grpcModelBindingData = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - var contentDict = GetTestContentDict(); + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var expectedBookList = new List() { new Book() { Name = "MyBook" } }; + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); - _mockBlobStorageConverter - .Setup(c => c.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName)) - .ReturnsAsync("test"); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - var result = await _mockBlobStorageConverter.Object.ConvertModelBindingDataAsync(contentDict, typeof(string), grpcModelBindingData); + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); - Assert.Equal(typeof(string), result.GetType()); + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (IEnumerable)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(pocoResult); + Assert.Equal(expectedBookList[0].Name, pocoResult.First().Name); } [Fact] - public async Task ToTargetTypeAsync_Works() - { - object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - byte[] byteArray = Encoding.UTF8.GetBytes("test"); - - _mockBlobStorageConverter.Setup(c => c.GetBlobStreamAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync(new MemoryStream(byteArray)); - _mockBlobStorageConverter.Setup(c => c.GetBlobBinaryDataAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync(byteArray); - _mockBlobStorageConverter.Setup(c => c.GetBlobStringAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync("test"); - _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); - _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); - _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); - _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); - _mockBlobStorageConverter.Setup(c => c.CreateBlobClient(Constants.Connection, Constants.ContainerName, Constants.BlobName)).Returns(new Mock().Object); - _mockBlobStorageConverter.Setup(c => c.CreateBlobContainerClient(Constants.Connection, Constants.ContainerName)).Returns(new Mock().Object); - - var streamResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(Stream), Constants.Connection, Constants.ContainerName, Constants.BlobName); - var byteArrayResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(Byte[]), Constants.Connection, Constants.ContainerName, Constants.BlobName); - var stringResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(string), Constants.Connection, Constants.ContainerName, Constants.BlobName); - var blobClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(BlobClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); - var blobBaseClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(BlobBaseClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); - var blockBlobClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(BlockBlobClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); - var pageBlobClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(PageBlobClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); - var appendBlobClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(AppendBlobClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); - var blobContainerClientResult = await _mockBlobStorageConverter.Object.ToTargetTypeAsync(typeof(BlobContainerClient), Constants.Connection, Constants.ContainerName, Constants.BlobName); - - Assert.Equal(typeof(MemoryStream), streamResult.GetType()); - Assert.Equal(typeof(Byte[]), byteArrayResult.GetType()); - Assert.Equal(typeof(string), stringResult.GetType()); - Assert.Equal(typeof(BlobClient), blobClientResult.GetType().BaseType); - Assert.Equal(typeof(BlobBaseClient), blobBaseClientResult.GetType().BaseType); - Assert.Equal(typeof(BlockBlobClient), blockBlobClientResult.GetType().BaseType); - Assert.Equal(typeof(PageBlobClient), pageBlobClientResult.GetType().BaseType); - Assert.Equal(typeof(AppendBlobClient), appendBlobClientResult.GetType().BaseType); - Assert.Equal(typeof(BlobContainerClient), blobContainerClientResult.GetType().BaseType); + public async Task ConvertAsync_ValidModelBindingData_POCOCollection_Array_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(Book[]), grpcModelBindingData); + + var expectedBookList = new Book[] { new Book() { Name = "MyBook" } }; + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (Book[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(pocoResult); + Assert.Equal(expectedBookList[0].Name, pocoResult.First().Name); } + // Unhappy cases + [Fact] - public void ToTargetTypeCollection_CloneToArray_Works() + public async Task ConvertAsync_ContentSource_AsObject_ReturnsUnhandled() { - IEnumerable stringCollection = new List() { "hello", "world"}; - string[] stringResult = (string[])_mockBlobStorageConverter.Object.ToTargetTypeCollection(stringCollection, nameof(BlobStorageConverter.CloneToArray), typeof(string)); + // Arrange + var context = new TestConverterContext(typeof(BlobClient), new Object()); - IEnumerable pocoCollection = new List() { new Book(), new Book() }; - Book[] pocoResult = (Book[])_mockBlobStorageConverter.Object.ToTargetTypeCollection(pocoCollection, nameof(BlobStorageConverter.CloneToArray), typeof(Book)); + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); - IEnumerable byteArraycollection = new List() { Encoding.UTF8.GetBytes("hello"), Encoding.UTF8.GetBytes("world") }; - Byte[][] byteArrayResult = (Byte[][])_mockBlobStorageConverter.Object.ToTargetTypeCollection(byteArraycollection, nameof(BlobStorageConverter.CloneToArray), typeof(Byte[])); + // Assert + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } - Assert.Equal(2, stringResult.Length); - Assert.Equal(typeof(string), stringResult[0].GetType()); + [Fact] + public async Task ConvertAsync_ModelBindingData_Null_ReturnsUnhandled() + { + // Arrange + var context = new TestConverterContext(typeof(BlobClient), null); - Assert.Equal(2, pocoResult.Length); - Assert.Equal(typeof(Book), pocoResult[0].GetType()); + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); - Assert.Equal(2, byteArrayResult.Length); - Assert.Equal(typeof(Byte[]), byteArrayResult[0].GetType()); + // Assert + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); } [Fact] - public void ToTargetTypeCollection_CloneToList_Works() + public async Task ConvertAsync_BlobClientIsNull_ReturnsUnhandled() { - IEnumerable stringCollection = new List() { "hello", "world" }; - IEnumerable stringResult = (IEnumerable)_mockBlobStorageConverter.Object.ToTargetTypeCollection(stringCollection, nameof(BlobStorageConverter.CloneToArray), typeof(string)); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - IEnumerable pocoCollection = new List() { new Book(), new Book() }; - IEnumerable pocoResult = (IEnumerable)_mockBlobStorageConverter.Object.ToTargetTypeCollection(pocoCollection, nameof(BlobStorageConverter.CloneToList), typeof(Book)); + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns((BlobClient)null); - IEnumerable byteArraycollection = new List() { Encoding.UTF8.GetBytes("hello"), Encoding.UTF8.GetBytes("world") }; - IEnumerable byteArrayResult = (IEnumerable)_mockBlobStorageConverter.Object.ToTargetTypeCollection(byteArraycollection, nameof(BlobStorageConverter.CloneToArray), typeof(Byte[])); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - Assert.Equal(2, stringResult.Count()); - Assert.Equal(typeof(string), stringResult.FirstOrDefault().GetType()); + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); - Assert.Equal(2, pocoResult.Count()); - Assert.Equal(typeof(Book), pocoResult.FirstOrDefault().GetType()); + // Assert + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } - Assert.Equal(2, byteArrayResult.Count()); - Assert.Equal(typeof(Byte[]), byteArrayResult.FirstOrDefault().GetType()); + [Fact] + public async Task ConvertAsync_ThrowsException_ReturnsFailure() + { + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + _mockBlobServiceClient + .Setup(m => m.GetBlobContainerClient(It.IsAny())) + .Throws(new Exception()); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); } + [Fact] + public async Task ConvertAsync_ModelBindingDataSource_NotBlobExtension_ReturnsUnhandled() + { + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "anotherExtensions"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingDataContentType_Unsupported_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs", contentType: "binary"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unexpected content-type. Currently only 'application/json' is supported.", conversionResult.Error.Message); + } [Fact] - public async Task DeserializeToTargetObjectAsync_CorrectPoco_Works() + public async Task ConvertAsync_ModelBindingData_InvalidContent_Throws_ReturnsFailed() { - object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - string jsonstr = "{" + "\"Id\" : \"1\", \"Title\" : \"title\", \"Author\" : \"author\"}"; - byte[] byteArray = Encoding.UTF8.GetBytes(jsonstr); + // Arrange + string badJsonData = $@"{{ + ""Connection"" : ""Connection"", + ""ContainerName"" ""ContainerName"", + ""BlobName"" : ""BlobName"", + }}"; - _mockBlobStorageConverter.Setup(c => c.GetBlobStreamAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync(new MemoryStream(byteArray)); + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(new BinaryData(badJsonData), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - var result = await _mockBlobStorageConverter.Object.DeserializeToTargetObjectAsync(typeof(Book), Constants.Connection, Constants.ContainerName, Constants.BlobName); + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); - Assert.Equal(typeof(Book), result.GetType()); + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); } [Fact] - public async Task DeserializeToTargetObjectAsync_IncorrectPoco_Fails() + public async Task ConvertAsync_MissingConnectionParameter_ReturnsFailed() { - object source = GetTestGrpcModelBindingData(GetFullTestBinaryData()); - string jsonstr = "{" + "\"Id\" : \"1\", \"Name\" : \"name\"}"; - byte[] byteArray = Encoding.UTF8.GetBytes(jsonstr); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(null), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - _mockBlobStorageConverter.Setup(c => c.GetBlobStreamAsync(Constants.Connection, Constants.ContainerName, Constants.BlobName)).ReturnsAsync(new MemoryStream(byteArray)); + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient)conversionResult.Value; - try - { - var result = await _mockBlobStorageConverter.Object.DeserializeToTargetObjectAsync(typeof(Book), Constants.Connection, Constants.ContainerName, Constants.BlobName); - Assert.Fail("Test fails as the expected exception not thrown"); - } - catch (Xunit.Sdk.FailException) { } + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Value cannot be null. (Parameter 'connectionName')", conversionResult.Error.Message); } - private BinaryData GetTestBinaryData() + [Fact] + public async Task ConvertAsync_MissingContainerNameParameter_ReturnsFailed() { - return new BinaryData("{" + "\"BlobName\" : \"BlobName\"" + "}"); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(container: null), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Value cannot be null. (Parameter 'containerName')", conversionResult.Error.Message); } - private BinaryData GetFullTestBinaryData() + [Fact] + public async Task ConvertAsync_BlobClient_MissingBlobNameParameter_ReturnsFailed() { - return new BinaryData("{" + - "\"Connection\" : \"Connection\"," + - "\"ContainerName\" : \"ContainerName\"," + - "\"BlobName\" : \"BlobName\"" + - "}"); + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(blobName: null), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Value cannot be null. (Parameter 'blobName')", conversionResult.Error.Message); } + [Fact] + public async Task ConvertAsync_POCO_InvalidJson_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book), grpcModelBindingData); + + var expectedBook = new Book() { Name = "MyBook" }; + + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name:\"MyBook\"}")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (Book)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + } - private GrpcModelBindingData GetTestGrpcModelBindingData(BinaryData binaryData) + private BinaryData GetTestBinaryData(string connection = "Connection", string container = "Container", string blobName = "MyBlob") { - return new GrpcModelBindingData(new ModelBindingData() - { - Version = "1.0", - Source = "AzureStorageBlobs", - Content = ByteString.CopyFrom(binaryData), - ContentType = "application/json" - }); + string jsonData = $@"{{ + ""Connection"" : ""{connection}"", + ""ContainerName"" : ""{container}"", + ""BlobName"" : ""{blobName}"" + }}"; + + return new BinaryData(jsonData); } - private GrpcCollectionModelBindingData GetTestGrpcCollectionModelBindingData() + private GrpcCollectionModelBindingData GetTestGrpcCollectionModelBindingData(BinaryData data) { var modelBindingData = new ModelBindingData() { Version = "1.0", Source = "AzureStorageBlobs", - Content = ByteString.CopyFrom(GetFullTestBinaryData()), + Content = ByteString.CopyFrom(data), ContentType = "application/json" }; @@ -400,14 +798,13 @@ private GrpcCollectionModelBindingData GetTestGrpcCollectionModelBindingData() return new GrpcCollectionModelBindingData(array); } - private Dictionary GetTestContentDict() + public class Book + { + public string Name { get; set; } + } + + public interface TMock { - return new Dictionary - { - { Constants.Connection, Constants.Connection }, - { Constants.ContainerName, Constants.ContainerName }, - { Constants.BlobName, Constants.BlobName } - }; } } } From 75717a76acae1919c6241c7a8779ec5ebef64e0a Mon Sep 17 00:00:00 2001 From: Fabio Cavalcante Date: Mon, 12 Jun 2023 14:27:33 -0700 Subject: [PATCH 19/47] Simplifying blob converter/avoid unnecessary reflection (#1614) * Simplifying blob converter/avoid unnecessary reflection * Reenabling disabled tests --- .../src/BlobStorageConverter.cs | 47 ++++++--------- .../Blob/BlobStorageConverterTests.cs | 60 +++++++++---------- 2 files changed, 47 insertions(+), 60 deletions(-) diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs index 30a5a8ce9..f5da47bda 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -4,8 +4,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; using Azure.Storage.Blobs; @@ -16,6 +14,7 @@ using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using System.Collections; namespace Microsoft.Azure.Functions.Worker { @@ -73,13 +72,17 @@ private async ValueTask ConvertFromBindingDataAsync(ConverterC private async ValueTask ConvertFromCollectionBindingDataAsync(ConverterContext context, CollectionModelBindingData collectionModelBindingData) { - var blobCollection = new List(collectionModelBindingData.ModelBindingDataArray.Length); - Type elementType = context.TargetType.IsArray ? context.TargetType.GetElementType() : context.TargetType.GenericTypeArguments[0]; + Type elementType = context.TargetType.IsArray + ? context.TargetType.GetElementType() + : context.TargetType.GenericTypeArguments[0]; + + IList result = Array.CreateInstance(elementType, collectionModelBindingData.ModelBindingDataArray.Length); try { - foreach (ModelBindingData modelBindingData in collectionModelBindingData.ModelBindingDataArray) + for (var i = 0; i < collectionModelBindingData.ModelBindingDataArray.Length; i++) { + var modelBindingData = collectionModelBindingData.ModelBindingDataArray[i]; if (!IsBlobExtension(modelBindingData)) { return ConversionResult.Unhandled(); @@ -90,12 +93,15 @@ private async ValueTask ConvertFromCollectionBindingDataAsync( if (element is not null) { - blobCollection.Add(element); + result[i] = element; } } - var methodName = context.TargetType.IsArray ? nameof(CloneToArray) : nameof(CloneToList); - var result = ToTargetTypeCollection(blobCollection, methodName, elementType); + if (!context.TargetType.IsArray) + { + var resultType = typeof(List<>).MakeGenericType(elementType); + result = (IList)Activator.CreateInstance(resultType, result); + } return ConversionResult.Success(result); } @@ -146,9 +152,9 @@ private Dictionary GetBindingDataContent(ModelBindingData bindin private async Task ToTargetTypeAsync(Type targetType, string connectionName, string containerName, string blobName) => targetType switch { - Type _ when targetType == typeof(String) => await GetBlobStringAsync(connectionName, containerName, blobName), + Type _ when targetType == typeof(string) => await GetBlobStringAsync(connectionName, containerName, blobName), Type _ when targetType == typeof(Stream) => await GetBlobStreamAsync(connectionName, containerName, blobName), - Type _ when targetType == typeof(Byte[]) => await GetBlobBinaryDataAsync(connectionName, containerName, blobName), + Type _ when targetType == typeof(byte[]) => await GetBlobBinaryDataAsync(connectionName, containerName, blobName), Type _ when targetType == typeof(BlobBaseClient) => CreateBlobClient(connectionName, containerName, blobName), Type _ when targetType == typeof(BlobClient) => CreateBlobClient(connectionName, containerName, blobName), Type _ when targetType == typeof(BlockBlobClient) => CreateBlobClient(connectionName, containerName, blobName), @@ -164,25 +170,6 @@ private Dictionary GetBindingDataContent(ModelBindingData bindin return _workerOptions?.Value?.Serializer?.Deserialize(content, targetType, CancellationToken.None); } - private object? ToTargetTypeCollection(IEnumerable blobCollection, string methodName, Type type) - { - blobCollection = blobCollection.Select(b => Convert.ChangeType(b, type)); - MethodInfo method = typeof(BlobStorageConverter).GetMethod(methodName, BindingFlags.Static | BindingFlags.NonPublic); - MethodInfo genericMethod = method.MakeGenericMethod(type); - - return genericMethod.Invoke(null, new[] { blobCollection.ToList() }); - } - - private static T[] CloneToArray(IList source) - { - return source.Cast().ToArray(); - } - - private static IEnumerable CloneToList(IList source) - { - return source.Cast(); - } - private async Task GetBlobStringAsync(string connectionName, string containerName, string blobName) { var client = CreateBlobClient(connectionName, containerName, blobName); @@ -190,7 +177,7 @@ private async Task GetBlobStringAsync(string connectionName, string cont return download.Value.Content.ToString(); } - private async Task GetBlobBinaryDataAsync(string connectionName, string containerName, string blobName) + private async Task GetBlobBinaryDataAsync(string connectionName, string containerName, string blobName) { using MemoryStream stream = new(); var client = CreateBlobClient(connectionName, containerName, blobName); diff --git a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs index d48a0f618..50ee3a78d 100644 --- a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs +++ b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs @@ -30,8 +30,8 @@ namespace Microsoft.Azure.Functions.WorkerExtension.Tests { public class BlobStorageConverterTests { - private BlobStorageConverter _blobStorageConverter; - private Mock _mockBlobServiceClient; + private readonly BlobStorageConverter _blobStorageConverter; + private readonly Mock _mockBlobServiceClient; public BlobStorageConverterTests() { @@ -77,7 +77,7 @@ public async Task ConvertAsync_ValidModelBindingData_BlobClient_ReturnsSuccess() Assert.Equal("MyBlob", clientResult.Name); } - [Fact (Skip = "Fails: ChangeType doesn't work for mock objects 'Object must implement IConvertible'")] + [Fact] public async Task ConvertAsync_ValidModelBindingData_BlobClientCollection_List_ReturnsSuccess() { // Arrange @@ -102,7 +102,7 @@ public async Task ConvertAsync_ValidModelBindingData_BlobClientCollection_List_R Assert.Equal("MyBlob", clientResult.First().Name); } - [Fact (Skip = "Fails: ChangeType doesn't work for mock objects 'Object must implement IConvertible'")] + [Fact] public async Task ConvertAsync_ValidModelBindingData_BlobClientCollection_Array_ReturnsSuccess() { // Arrange @@ -148,7 +148,7 @@ public async Task ConvertAsync_ValidModelBindingData_BlobContainerClient_Returns Assert.Equal("MyContainer", clientResult.Name); } - [Fact (Skip = "Fails: ChangeType doesn't work for mock objects 'Object must implement IConvertible'")] + [Fact] public async Task ConvertAsync_ValidModelBindingData_BlobContainerClientCollection_List_ReturnsSuccess() { // Arrange @@ -170,7 +170,7 @@ public async Task ConvertAsync_ValidModelBindingData_BlobContainerClientCollecti Assert.Equal("MyContainer", clientResult.First().Name); } - [Fact (Skip = "Fails: ChangeType doesn't work for mock objects 'Object must implement IConvertible'")] + [Fact] public async Task ConvertAsync_ValidModelBindingData_BlobContainerClientCollection_Array_ReturnsSuccess() { // Arrange @@ -197,7 +197,7 @@ public async Task ConvertAsync_ValidModelBindingData_String_ReturnsSuccess() { // Arrange var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(String), grpcModelBindingData); + var context = new TestConverterContext(typeof(string), grpcModelBindingData); var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); var mockResponse = new Mock>(); @@ -213,7 +213,7 @@ public async Task ConvertAsync_ValidModelBindingData_String_ReturnsSuccess() // Act var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var stringResult = (String)conversionResult.Value; + var stringResult = (string)conversionResult.Value; // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); @@ -225,7 +225,7 @@ public async Task ConvertAsync_ValidModelBindingData_StringCollection_List_Retur { // Arrange var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); var mockResponse = new Mock>(); @@ -241,11 +241,11 @@ public async Task ConvertAsync_ValidModelBindingData_StringCollection_List_Retur // Act var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var stringResult = (IEnumerable)conversionResult.Value; + var stringResult = (IEnumerable)conversionResult.Value; // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsAssignableFrom>(stringResult); + Assert.IsAssignableFrom>(stringResult); Assert.Equal("MyBlobString", stringResult.First().ToString()); } @@ -254,7 +254,7 @@ public async Task ConvertAsync_ValidModelBindingData_StringCollection_Array_Retu { // Arrange var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(String[]), grpcModelBindingData); + var context = new TestConverterContext(typeof(string[]), grpcModelBindingData); var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); var mockResponse = new Mock>(); @@ -270,11 +270,11 @@ public async Task ConvertAsync_ValidModelBindingData_StringCollection_Array_Retu // Act var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var stringResult = (String[])conversionResult.Value; + var stringResult = (string[])conversionResult.Value; // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(stringResult); + Assert.IsType(stringResult); Assert.Equal("MyBlobString", stringResult.First().ToString()); } @@ -309,7 +309,7 @@ public async Task ConvertAsync_ValidModelBindingData_Stream_ReturnsSuccess() Assert.Equal(expectedStream, streamResult); } - [Fact (Skip = "Fails - broken scenario. ChangeType fails trying to convert MemoryStream to Stream) - 'Object must implement IConvertible'")] + [Fact] public async Task ConvertAsync_ValidModelBindingData_StreamCollection_List_ReturnsSuccess() { // Arrange @@ -342,7 +342,7 @@ public async Task ConvertAsync_ValidModelBindingData_StreamCollection_List_Retur Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); } - [Fact (Skip = "Fails - broken scenario. ChangeType fails trying to convert MemoryStream to Stream) - 'Object must implement IConvertible'")] + [Fact] public async Task ConvertAsync_ValidModelBindingData_StreamCollection_Array_ReturnsSuccess() { // Arrange @@ -371,8 +371,6 @@ public async Task ConvertAsync_ValidModelBindingData_StreamCollection_Array_Retu // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(streamResult); - Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); } @@ -381,11 +379,13 @@ public async Task ConvertAsync_ValidModelBindingData_ByteArray_ReturnsSuccess() { // Arrange var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(Byte[]), grpcModelBindingData); + var context = new TestConverterContext(typeof(byte[]), grpcModelBindingData); var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); - var testStream = new MemoryStream(expectedByteArray); - testStream.Position = 0; + var testStream = new MemoryStream(expectedByteArray) + { + Position = 0 + }; var mockBlobClient = new Mock(); mockBlobClient @@ -400,11 +400,11 @@ public async Task ConvertAsync_ValidModelBindingData_ByteArray_ReturnsSuccess() // Act var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var byteResult = (Byte[])conversionResult.Value; + var byteResult = (byte[])conversionResult.Value; // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(byteResult); + Assert.IsType(byteResult); Assert.Equal(new byte[0], byteResult); // Assert.Equal(expectedByteArray, byteResult); // Running into issues mocking DownloadToAsync stream update } @@ -414,7 +414,7 @@ public async Task ConvertAsync_ValidModelBindingData_ByteArrayCollection_List_Re { // Arrange var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); var testStream = new MemoryStream(expectedByteArray); @@ -433,11 +433,11 @@ public async Task ConvertAsync_ValidModelBindingData_ByteArrayCollection_List_Re // Act var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var byteResult = (IEnumerable)conversionResult.Value; + var byteResult = (IEnumerable)conversionResult.Value; // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsAssignableFrom>(byteResult); + Assert.IsAssignableFrom>(byteResult); // Assert.Equal(expectedByteArray, byteResult.First()); // Running into issues mocking DownloadToAsync stream update } @@ -446,7 +446,7 @@ public async Task ConvertAsync_ValidModelBindingData_ByteArrayCollection_Array_R { // Arrange var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(Byte[][]), grpcModelBindingData); + var context = new TestConverterContext(typeof(byte[][]), grpcModelBindingData); var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); var testStream = new MemoryStream(expectedByteArray); @@ -465,11 +465,11 @@ public async Task ConvertAsync_ValidModelBindingData_ByteArrayCollection_Array_R // Act var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var byteResult = (Byte[][])conversionResult.Value; + var byteResult = (byte[][])conversionResult.Value; // Assert Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(byteResult); + Assert.IsType(byteResult); // Assert.Equal(expectedByteArray, byteResult.First()); // Running into issues mocking DownloadToAsync steam update } @@ -581,7 +581,7 @@ public async Task ConvertAsync_ValidModelBindingData_POCOCollection_Array_Return public async Task ConvertAsync_ContentSource_AsObject_ReturnsUnhandled() { // Arrange - var context = new TestConverterContext(typeof(BlobClient), new Object()); + var context = new TestConverterContext(typeof(BlobClient), new object()); // Act var conversionResult = await _blobStorageConverter.ConvertAsync(context); From 175081ba99c349abc5403b3ce260cbfa2670e2e0 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari <37918412+aishwaryabh@users.noreply.github.com> Date: Mon, 19 Jun 2023 13:14:17 -0700 Subject: [PATCH 20/47] [SDK TypeBinding] Improve error logging for failed deserialization for POCO scenarios (#1639) * commiting changes * changed name --- .../src/BlobStorageConverter.cs | 13 +++++++ .../Blob/BlobStorageConverterTests.cs | 34 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs index f5da47bda..154469afd 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -15,6 +15,8 @@ using Microsoft.Extensions.Logging; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; using System.Collections; +using System.Text.Json; +using System.Globalization; namespace Microsoft.Azure.Functions.Worker { @@ -105,6 +107,17 @@ private async ValueTask ConvertFromCollectionBindingDataAsync( return ConversionResult.Success(result); } + catch (JsonException ex) + { + string msg = String.Format(CultureInfo.CurrentCulture, + @"Binding parameters to complex objects uses JSON serialization. + 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or + 2. Change the blob to be valid json. + The JSON parser failed: {0}", + ex.Message); + + return ConversionResult.Failed(new InvalidOperationException(msg, ex)); + } catch (Exception ex) { return ConversionResult.Failed(ex); diff --git a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs index 50ee3a78d..53ffb54c0 100644 --- a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs +++ b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs @@ -771,6 +771,40 @@ public async Task ConvertAsync_POCO_InvalidJson_ReturnsFailed() Assert.IsType(conversionResult.Error); } + [Fact] + public async Task ConvertAsync_IncorrectJsonContent_POCOCollection_Array_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); + var context = new TestConverterContext(typeof(Book[]), grpcModelBindingData); + + var expectedBookList = new Book[] { new Book() { Name = "MyBook" } }; + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("i should fail")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (Book[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.IsType(conversionResult.Error.InnerException); + } + private BinaryData GetTestBinaryData(string connection = "Connection", string container = "Container", string blobName = "MyBlob") { string jsonData = $@"{{ From bbf7bf69f5989ca0401d250b729bd924ef9c91ce Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Mon, 17 Jul 2023 16:02:12 -0700 Subject: [PATCH 21/47] Move shared code to Extensions.Shared project (#1716) --- .../Configuration/ConfigurationExtensions.cs | 43 ++++++++++++++++++- .../InvalidBindingSourceException.cs | 28 ++++++++++++ .../Exceptions/InvalidContentTypeException.cs | 28 ++++++++++++ .../src/BlobStorageConverter.cs | 38 ++++++---------- .../Config/BlobStorageBindingOptionsSetup.cs | 37 +--------------- .../Worker.Extensions.Storage.Blobs.csproj | 1 - .../FunctionMetadataGeneratorTests.cs | 12 +++--- .../StorageBindingTests.cs | 4 +- test/SdkE2ETests/PublishTests.cs | 9 ++-- .../Blob/BlobStorageConverterTests.cs | 25 ++--------- 10 files changed, 125 insertions(+), 100 deletions(-) create mode 100644 extensions/Worker.Extensions.Shared/Exceptions/InvalidBindingSourceException.cs create mode 100644 extensions/Worker.Extensions.Shared/Exceptions/InvalidContentTypeException.cs diff --git a/extensions/Worker.Extensions.Shared/Configuration/ConfigurationExtensions.cs b/extensions/Worker.Extensions.Shared/Configuration/ConfigurationExtensions.cs index cff348de4..bd2a95a75 100644 --- a/extensions/Worker.Extensions.Shared/Configuration/ConfigurationExtensions.cs +++ b/extensions/Worker.Extensions.Shared/Configuration/ConfigurationExtensions.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; +using System.Globalization; using Microsoft.Extensions.Configuration; namespace Microsoft.Azure.Functions.Worker.Extensions @@ -30,6 +32,45 @@ internal static IConfigurationSection GetWebJobsConnectionStringSection(this ICo return section; } + /// + /// Either constructs the serviceUri from the provided accountName + /// or retrieves the serviceUri for the specific resource (i.e. blobServiceUri or queueServiceUri) + /// + /// configuration section for a given connection name + /// The subdomain of the serviceUri (i.e. blob, queue, table) + /// The serviceUri for the specific resource (i.e. blobServiceUri or queueServiceUri) + internal static bool TryGetServiceUriForStorageAccounts(this IConfiguration configuration, string subDomain, out Uri serviceUri) + { + if (subDomain is null) + { + throw new ArgumentNullException(nameof(subDomain)); + } + var serviceUriConfig = string.Format(CultureInfo.InvariantCulture, "{0}ServiceUri", subDomain); + + if (configuration.GetValue("accountName") is { } accountName) + { + serviceUri = FormatServiceUri(accountName, subDomain); + return true; + } + else if (configuration.GetValue($"{subDomain}ServiceUri") is { } uriStr) + { + serviceUri = new Uri(uriStr); + return true; + } + + serviceUri = default(Uri)!; + return false; + } + + /// + /// Generates the serviceUri for a particular storage resource + /// + private static Uri FormatServiceUri(string accountName, string subDomain, string defaultProtocol = "https", string endpointSuffix = "core.windows.net") + { + var uri = string.Format(CultureInfo.InvariantCulture, "{0}://{1}.{2}.{3}", defaultProtocol, accountName, subDomain, endpointSuffix); + return new Uri(uri); + } + /// /// Creates a WebJobs specific prefixed string using a given connection name. /// @@ -58,4 +99,4 @@ private static IConfigurationSection GetConnectionStringOrSetting(this IConfigur return configuration.GetSection(connectionName); } } -} \ No newline at end of file +} diff --git a/extensions/Worker.Extensions.Shared/Exceptions/InvalidBindingSourceException.cs b/extensions/Worker.Extensions.Shared/Exceptions/InvalidBindingSourceException.cs new file mode 100644 index 000000000..f0bbee0b6 --- /dev/null +++ b/extensions/Worker.Extensions.Shared/Exceptions/InvalidBindingSourceException.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.Functions.Worker.Extensions +{ + /// + /// The exception that is thrown when an invalid binding source is provided. + /// + internal class InvalidBindingSourceException : InvalidOperationException + { + /// + /// Initializes a new instance of the InvalidBindingSourceException class with a specified error message. + /// + /// The source(s) that is supported. + public InvalidBindingSourceException(string source) : base($"Unexpected binding source. Only '{source}' is supported.") { } + + /// + /// Initializes a new instance of the InvalidBindingSourceException class with a specified error message + /// and a reference to the inner exception that is the cause of this exception. + /// + /// The source(s) that is supported. + /// The exception that is the cause of the current exception + /// or a null reference if no inner exception is specified.. + public InvalidBindingSourceException(string source, Exception? innerException) : base($"Unexpected binding source. Only '{source}' is supported.", innerException) { } + } +} diff --git a/extensions/Worker.Extensions.Shared/Exceptions/InvalidContentTypeException.cs b/extensions/Worker.Extensions.Shared/Exceptions/InvalidContentTypeException.cs new file mode 100644 index 000000000..abfbb4999 --- /dev/null +++ b/extensions/Worker.Extensions.Shared/Exceptions/InvalidContentTypeException.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.Functions.Worker.Extensions +{ + /// + /// The exception that is thrown when an invalid content-type is provided. + /// + internal class InvalidContentTypeException : InvalidOperationException + { + /// + /// Initializes a new instance of the InvalidContentTypeException class with a specified error message. + /// + /// The content type(s) that is supported. + public InvalidContentTypeException(string contentType) : base($"Unexpected content-type. Only '{contentType}' is supported.") { } + + /// + /// Initializes a new instance of the InvalidContentTypeException class with a specified error message + /// and a reference to the inner exception that is the cause of this exception. + /// + /// The content type(s) that is supported. + /// The exception that is the cause of the current exception + /// or a null reference if no inner exception is specified.. + public InvalidContentTypeException(string contentType, Exception innerException) : base($"Unexpected content-type. Only '{contentType}' is supported.", innerException) { } + } +} diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs index 154469afd..5bfe9b7f0 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -10,10 +10,11 @@ using Azure.Storage.Blobs.Specialized; using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Azure.Functions.Worker.Extensions; +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; using Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs; using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging; -using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; using System.Collections; using System.Text.Json; using System.Globalization; @@ -49,27 +50,22 @@ public async ValueTask ConvertAsync(ConverterContext context) private async ValueTask ConvertFromBindingDataAsync(ConverterContext context, ModelBindingData modelBindingData) { - if (!IsBlobExtension(modelBindingData)) - { - return ConversionResult.Unhandled(); - } - try { + if (modelBindingData.Source is not Constants.BlobExtensionName) + { + throw new InvalidBindingSourceException(Constants.BlobExtensionName); + } + Dictionary content = GetBindingDataContent(modelBindingData); var result = await ConvertModelBindingDataAsync(content, context.TargetType, modelBindingData); - if (result is not null) - { - return ConversionResult.Success(result); - } + return ConversionResult.Success(result); } catch (Exception ex) { return ConversionResult.Failed(ex); } - - return ConversionResult.Unhandled(); } private async ValueTask ConvertFromCollectionBindingDataAsync(ConverterContext context, CollectionModelBindingData collectionModelBindingData) @@ -85,9 +81,10 @@ private async ValueTask ConvertFromCollectionBindingDataAsync( for (var i = 0; i < collectionModelBindingData.ModelBindingDataArray.Length; i++) { var modelBindingData = collectionModelBindingData.ModelBindingDataArray[i]; - if (!IsBlobExtension(modelBindingData)) + + if (modelBindingData.Source is not Constants.BlobExtensionName) { - return ConversionResult.Unhandled(); + throw new InvalidBindingSourceException(Constants.BlobExtensionName); } Dictionary content = GetBindingDataContent(modelBindingData); @@ -124,23 +121,12 @@ 2. Change the blob to be valid json. } } - private bool IsBlobExtension(ModelBindingData bindingData) - { - if (bindingData?.Source is not Constants.BlobExtensionName) - { - _logger.LogTrace("Source '{source}' is not supported by {converter}", bindingData?.Source, nameof(BlobStorageConverter)); - return false; - } - - return true; - } - private Dictionary GetBindingDataContent(ModelBindingData bindingData) { return bindingData?.ContentType switch { Constants.JsonContentType => new Dictionary(bindingData?.Content?.ToObjectFromJson>(), StringComparer.OrdinalIgnoreCase), - _ => throw new NotSupportedException($"Unexpected content-type. Currently only '{Constants.JsonContentType}' is supported.") + _ => throw new InvalidContentTypeException(Constants.JsonContentType) }; } diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptionsSetup.cs b/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptionsSetup.cs index 654653702..3847c00ce 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptionsSetup.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptionsSetup.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.Options; using Microsoft.Azure.Functions.Worker.Extensions; using Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs; -using System.Globalization; namespace Microsoft.Azure.Functions.Worker { @@ -52,7 +51,7 @@ public void Configure(string name, BlobStorageBindingOptions options) } else { - if (TryGetServiceUri(connectionSection, out Uri serviceUri)) + if (connectionSection.TryGetServiceUriForStorageAccounts(BlobServiceUriSubDomain, out Uri serviceUri)) { options.ServiceUri = serviceUri; } @@ -61,39 +60,5 @@ public void Configure(string name, BlobStorageBindingOptions options) options.BlobClientOptions = (BlobClientOptions)_componentFactory.CreateClientOptions(typeof(BlobClientOptions), null, connectionSection); options.Credential = _componentFactory.CreateTokenCredential(connectionSection); } - - /// - /// Either constructs the serviceUri from the provided accountName - /// or retrieves the serviceUri for the specific resource (i.e. blobServiceUri or queueServiceUri) - /// - private bool TryGetServiceUri(IConfiguration configuration, out Uri serviceUri) - { - var serviceUriConfig = string.Format(CultureInfo.InvariantCulture, "{0}ServiceUri", BlobServiceUriSubDomain); - - string accountName; - string uriStr; - if ((accountName = configuration.GetValue("accountName")) is not null) - { - serviceUri = FormatServiceUri(accountName); - return true; - } - else if ((uriStr = configuration.GetValue(serviceUriConfig)) is not null) - { - serviceUri = new Uri(uriStr); - return true; - } - - serviceUri = default(Uri)!; - return false; - } - - /// - /// Generates the serviceUri for a particular storage resource - /// - private Uri FormatServiceUri(string accountName, string defaultProtocol = "https", string endpointSuffix = "core.windows.net") - { - var uri = string.Format(CultureInfo.InvariantCulture, "{0}://{1}.{2}.{3}", defaultProtocol, accountName, BlobServiceUriSubDomain, endpointSuffix); - return new Uri(uri); - } } } \ No newline at end of file diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Worker.Extensions.Storage.Blobs.csproj b/extensions/Worker.Extensions.Storage.Blobs/src/Worker.Extensions.Storage.Blobs.csproj index 67f95ee29..ac67a29a9 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Worker.Extensions.Storage.Blobs.csproj +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Worker.Extensions.Storage.Blobs.csproj @@ -23,7 +23,6 @@ - diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 57e64ae30..2dd9dad39 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -171,8 +171,8 @@ public void StorageFunctions() AssertDictionary(extensions, new Dictionary { - { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.2" }, - { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.2" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.3" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.3" }, }); void ValidateQueueTrigger(ExpandoObject b) @@ -266,7 +266,7 @@ public void BlobStorageFunctions_SDKTypeBindings() AssertDictionary(extensions, new Dictionary { - { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.0-beta.1" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.3" }, }); var blobClientToBlobStringFunction = functions.Single(p => p.Name == "BlobClientToBlobStringFunction"); @@ -353,7 +353,7 @@ public void BlobCollectionFunctions_SDKTypeBindings() AssertDictionary(extensions, new Dictionary { - { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.0-beta.1" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.3" }, }); var blobStringToBlobClientEnumerable = functions.Single(p => p.Name == "BlobStringToBlobClientEnumerable"); @@ -487,8 +487,8 @@ public void MultiOutput_OnReturnType() AssertDictionary(extensions, new Dictionary { - { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.2" }, - { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.2" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.3" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.3" }, }); void ValidateQueueTrigger(ExpandoObject b) diff --git a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs index c0d0ff211..c66ac9058 100644 --- a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs +++ b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs @@ -206,7 +206,7 @@ public Task> GetFunctionMetadataAsync(string d metadataList.Add(Function0); var Function1RawBindings = new List(); Function1RawBindings.Add(@"{""name"":""$return"",""type"":""Queue"",""direction"":""Out"",""queueName"":""queue2""}"); - Function1RawBindings.Add(@"{""name"":""blob"",""type"":""BlobTrigger"",""direction"":""In"",""path"":""container2/%file%"",""dataType"":""String""}"); + Function1RawBindings.Add(@"{""name"":""blob"",""type"":""BlobTrigger"",""direction"":""In"",""properties"":{""supportsDeferredBinding"":""True""},""path"":""container2/%file%"",""dataType"":""String""}"); var Function1 = new DefaultFunctionMetadata { @@ -225,7 +225,7 @@ public Task> GetFunctionMetadataAsync(string d { Language = "dotnet-isolated", Name = "BlobsToQueueFunction", - EntryPoint = "TestProject.QueueTriggerAndOutput.BlobsToQueue", + EntryPoint = "FunctionApp.QueueTriggerAndOutput.BlobsToQueue", RawBindings = Function2RawBindings, ScriptFile = "TestProject.dll" }; diff --git a/test/SdkE2ETests/PublishTests.cs b/test/SdkE2ETests/PublishTests.cs index a69130b61..b7d9feb45 100644 --- a/test/SdkE2ETests/PublishTests.cs +++ b/test/SdkE2ETests/PublishTests.cs @@ -64,7 +64,7 @@ private async Task RunPublishTest(string outputDir, string additionalParams = nu "Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.Startup, Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader, Version=1.0.0.0, Culture=neutral, PublicKeyToken=551316b6919f366c", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.dll"), new Extension("AzureStorageBlobs", - "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageBlobsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Blobs, Version=5.1.0.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", + "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageBlobsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Blobs, Version=5.1.3.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.dll"), new Extension("AzureStorageQueues", "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageQueuesWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Queues, Version=5.1.2.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", @@ -123,11 +123,8 @@ private async Task RunPublishTestForSdkTypeBindings(string outputDir, string add "Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.Startup, Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader, Version=1.0.0.0, Culture=neutral, PublicKeyToken=551316b6919f366c", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.dll"), new Extension("AzureStorageBlobs", - "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageBlobsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Blobs, Version=5.1.0.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", - @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.dll"), - new Extension("AzureStorageQueues", - "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageQueuesWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Queues, Version=5.0.0.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", - @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Queues.dll") + "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageBlobsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Blobs, Version=5.1.3.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", + @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.dll") } }); Assert.True(JToken.DeepEquals(extensionsJsonContents, expected), $"Actual: {extensionsJsonContents}{Environment.NewLine}Expected: {expected}"); diff --git a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs index 53ffb54c0..5474961df 100644 --- a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs +++ b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs @@ -603,25 +603,6 @@ public async Task ConvertAsync_ModelBindingData_Null_ReturnsUnhandled() Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); } - [Fact] - public async Task ConvertAsync_BlobClientIsNull_ReturnsUnhandled() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns((BlobClient)null); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - - // Assert - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); - } - [Fact] public async Task ConvertAsync_ThrowsException_ReturnsFailure() { @@ -641,7 +622,7 @@ public async Task ConvertAsync_ThrowsException_ReturnsFailure() } [Fact] - public async Task ConvertAsync_ModelBindingDataSource_NotBlobExtension_ReturnsUnhandled() + public async Task ConvertAsync_ModelBindingDataSource_NotBlobExtension_ReturnsFailed() { // Arrange var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "anotherExtensions"); @@ -651,7 +632,7 @@ public async Task ConvertAsync_ModelBindingDataSource_NotBlobExtension_ReturnsUn var conversionResult = await _blobStorageConverter.ConvertAsync(context); // Assert - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); } [Fact] @@ -666,7 +647,7 @@ public async Task ConvertAsync_ModelBindingDataContentType_Unsupported_ReturnsFa // Assert Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.Equal("Unexpected content-type. Currently only 'application/json' is supported.", conversionResult.Error.Message); + Assert.Equal("Unexpected content-type. Only 'application/json' is supported.", conversionResult.Error.Message); } [Fact] From 518edf3dc69d9124b2cec77bc9e7a27d58fdf092 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Mon, 10 Jul 2023 16:36:32 -0700 Subject: [PATCH 22/47] Refactor deferred binding attributes (#1723) --- .../src/BlobInputAttribute.cs | 2 +- .../src/BlobStorageConverter.cs | 6 +++--- .../src/BlobTriggerAttribute.cs | 2 +- .../FunctionMetadataProviderGenerator.Parser.cs | 2 +- .../Features/DefaultInputConversionFeatureTests.cs | 4 ++-- test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs | 5 ++--- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs index 89e4ca58c..2180081cd 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs @@ -6,7 +6,7 @@ namespace Microsoft.Azure.Functions.Worker { - [AllowConverterFallback(false)] + [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] [InputConverter(typeof(BlobStorageConverter))] public sealed class BlobInputAttribute : InputBindingAttribute { diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs index 5bfe9b7f0..116e0fd07 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -74,13 +74,13 @@ private async ValueTask ConvertFromCollectionBindingDataAsync( ? context.TargetType.GetElementType() : context.TargetType.GenericTypeArguments[0]; - IList result = Array.CreateInstance(elementType, collectionModelBindingData.ModelBindingDataArray.Length); + IList result = Array.CreateInstance(elementType, collectionModelBindingData.ModelBindingData.Length); try { - for (var i = 0; i < collectionModelBindingData.ModelBindingDataArray.Length; i++) + for (var i = 0; i < collectionModelBindingData.ModelBindingData.Length; i++) { - var modelBindingData = collectionModelBindingData.ModelBindingDataArray[i]; + var modelBindingData = collectionModelBindingData.ModelBindingData[i]; if (modelBindingData.Source is not Constants.BlobExtensionName) { diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs index 6313dfe48..8231635c9 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs @@ -6,7 +6,7 @@ namespace Microsoft.Azure.Functions.Worker { - [AllowConverterFallback(true)] + [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] [InputConverter(typeof(BlobStorageConverter))] public sealed class BlobTriggerAttribute : TriggerBindingAttribute { diff --git a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Parser.cs b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Parser.cs index dcf3674f3..2e6cf4ecc 100644 --- a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Parser.cs +++ b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Parser.cs @@ -384,7 +384,7 @@ private bool DoesConverterSupportDeferredBinding(TypedConstant converter, string if (!converterAdvertisesTypes) { - // If a converter advertises deferred binding but does not explictly advertise any types then DeferredBinding will be supported for all the types + // If a converter advertises deferred binding but does not explicitly advertise any types then DeferredBinding will be supported for all the types return true; } diff --git a/test/DotNetWorkerTests/Features/DefaultInputConversionFeatureTests.cs b/test/DotNetWorkerTests/Features/DefaultInputConversionFeatureTests.cs index 5ec11b5e0..f71716975 100644 --- a/test/DotNetWorkerTests/Features/DefaultInputConversionFeatureTests.cs +++ b/test/DotNetWorkerTests/Features/DefaultInputConversionFeatureTests.cs @@ -208,7 +208,7 @@ public async Task Convert_Using_Advertised_Converter_Specified_In_ConverterConte IReadOnlyDictionary properties = new Dictionary() { { PropertyBagKeys.BindingAttributeSupportedConverters, new Dictionary>() { { - typeof(MySimpleSyncInputConverter2), new List() } } }, + typeof(MySimpleSyncInputConverter2), new List() } } }, { PropertyBagKeys.ConverterFallbackBehavior, ConverterFallbackBehavior.Disallow } }; var converterContext = CreateConverterContext(typeof(Poco[]), "0c67c078-7213-4e91-ad41-f8747c865f3d", properties); @@ -225,7 +225,7 @@ public async Task Convert_Using_Advertised_Converter_Specified_In_ConverterConte IReadOnlyDictionary properties = new Dictionary() { { PropertyBagKeys.BindingAttributeSupportedConverters, new Dictionary>() { { - typeof(MySimpleSyncInputConverter2), null } } }, + typeof(MySimpleSyncInputConverter2), null } } }, { PropertyBagKeys.ConverterFallbackBehavior, ConverterFallbackBehavior.Disallow } }; var converterContext = CreateConverterContext(typeof(IEnumerable), "0c67c078-7213-4e91-ad41-f8747c865f3d", properties); diff --git a/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs b/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs index 34dc65348..0c1ebca8f 100644 --- a/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs +++ b/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs @@ -121,9 +121,9 @@ public void GrpcFunctionDefinition_BlobInput_Creates() { Assert.Equal("myBlob", q.Name); Assert.Equal(typeof(string), q.Type); - Assert.Contains(PropertyBagKeys.AllowConverterFallback, q.Properties.Keys); + Assert.Contains(PropertyBagKeys.ConverterFallbackBehavior, q.Properties.Keys); Assert.Contains(PropertyBagKeys.BindingAttributeSupportedConverters, q.Properties.Keys); - Assert.True(true, q.Properties[PropertyBagKeys.AllowConverterFallback].ToString()); + Assert.Equal("Default", q.Properties[PropertyBagKeys.ConverterFallbackBehavior].ToString()); Assert.Contains(new Dictionary>().ToString(), q.Properties[PropertyBagKeys.BindingAttributeSupportedConverters].ToString()); }); @@ -146,7 +146,6 @@ public void GrpcFunctionDefinition_BlobInput_Creates() }); } - private class MyFunctionClass { public HttpResponseData Run(HttpRequestData req) From 88bf6104539fef054267d63aad71c6f07cdea51f Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Thu, 13 Jul 2023 16:55:51 -0700 Subject: [PATCH 23/47] Infer collection bindings using the blobPath (#1690) --- .../release_notes.md | 7 +- .../src/BlobInputAttribute.cs | 14 +- .../src/BlobStorageConverter.cs | 226 +++-- .../src/BlobTriggerAttribute.cs | 2 +- .../src/Constants.cs | 3 - .../src/Properties/AssemblyInfo.cs | 2 +- .../Worker.Extensions.Storage.Blobs.csproj | 6 +- .../Blob/BlobInputBindingSamples.cs | 131 +-- .../Blob/BlobTriggerBindingSamples.cs | 30 + .../Blob/ExpressionFunction.cs | 28 - samples/WorkerBindingSamples/Program.cs | 5 + .../WorkerBindingSamples.csproj | 10 +- .../Properties/AssemblyInfo.cs | 2 +- .../Properties/AssemblyInfo.cs | 2 +- .../E2EApp/Blob/BlobInputBindingFunctions.cs | 254 +++++- .../Blob/BlobTriggerBindingFunctions.cs | 11 - test/E2ETests/E2ETests/E2ETests.csproj | 2 +- .../E2ETests/Helpers/StorageHelpers.cs | 12 +- .../E2ETests/Storage/BlobEndToEndTests.cs | 158 ++-- .../FunctionMetadataGeneratorTests.cs | 22 +- .../IntegratedTriggersAndBindingsTests.cs | 4 +- .../StorageBindingTests.cs | 49 +- .../functions.metadata | 514 +---------- test/SdkE2ETests/Contents/functions.metadata | 2 - .../Blob/BlobClientTests.cs | 222 +++++ .../Blob/BlobContainerClientTests.cs | 149 ++++ .../Blob/BlobStorageConverterCoreTests.cs | 191 ++++ .../Blob/BlobStorageConverterTests.cs | 825 ------------------ .../Blob/BlobTestHelper.cs | 21 + .../Blob/ByteArrayTests.cs | 319 +++++++ .../Worker.Extensions.Tests/Blob/POCOTests.cs | 368 ++++++++ .../Blob/StreamTests.cs | 279 ++++++ .../Blob/StringTests.cs | 268 ++++++ .../Worker.Extensions.Tests/GrpcTestHelper.cs | 25 + 34 files changed, 2513 insertions(+), 1650 deletions(-) delete mode 100644 samples/WorkerBindingSamples/Blob/ExpressionFunction.cs create mode 100644 test/Worker.Extensions.Tests/Blob/BlobClientTests.cs create mode 100644 test/Worker.Extensions.Tests/Blob/BlobContainerClientTests.cs create mode 100644 test/Worker.Extensions.Tests/Blob/BlobStorageConverterCoreTests.cs delete mode 100644 test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs create mode 100644 test/Worker.Extensions.Tests/Blob/BlobTestHelper.cs create mode 100644 test/Worker.Extensions.Tests/Blob/ByteArrayTests.cs create mode 100644 test/Worker.Extensions.Tests/Blob/POCOTests.cs create mode 100644 test/Worker.Extensions.Tests/Blob/StreamTests.cs create mode 100644 test/Worker.Extensions.Tests/Blob/StringTests.cs create mode 100644 test/Worker.Extensions.Tests/GrpcTestHelper.cs diff --git a/extensions/Worker.Extensions.Storage.Blobs/release_notes.md b/extensions/Worker.Extensions.Storage.Blobs/release_notes.md index f13105d3c..fe008fa46 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/release_notes.md +++ b/extensions/Worker.Extensions.Storage.Blobs/release_notes.md @@ -4,7 +4,8 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs +### Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs 6.0.0 -- Add support for SDK-type bindings via deferred binding feature #1108 -- Assign cardinality correctly for Blob collection scenarios #1271 +- Add support for SDK-type bindings via deferred binding feature +- Remove IsBatched property from BlobInput binding attribute +- Infer binding cardinality based on blobPath diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs index 2180081cd..a19b78ad3 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobInputAttribute.cs @@ -6,14 +6,12 @@ namespace Microsoft.Azure.Functions.Worker { - [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] [InputConverter(typeof(BlobStorageConverter))] + [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] public sealed class BlobInputAttribute : InputBindingAttribute { private readonly string _blobPath; - private bool _isBatched = false; - /// Initializes a new instance of the class. /// The path of the blob to which to bind. public BlobInputAttribute(string blobPath) @@ -29,16 +27,6 @@ public string BlobPath get { return _blobPath; } } - /// - /// Gets or sets the configuration to enable batch processing of blobs. Default value is "false". - /// - [DefaultValue(false)] - public bool IsBatched - { - get => _isBatched; - set => _isBatched = value; - } - /// /// Gets or sets the app setting name that contains the Azure Storage connection string. /// diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs index 116e0fd07..a290d8a92 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -2,11 +2,16 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections; using System.Collections.Generic; +using System.Globalization; using System.IO; +using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Core; @@ -15,9 +20,6 @@ using Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs; using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging; -using System.Collections; -using System.Text.Json; -using System.Globalization; namespace Microsoft.Azure.Functions.Worker { @@ -30,6 +32,7 @@ internal class BlobStorageConverter : IInputConverter private readonly IOptions _workerOptions; private readonly IOptionsSnapshot _blobOptions; private readonly ILogger _logger; + private readonly Regex BlobIsFileRegex = new Regex(@"\.[^.\/]+$"); public BlobStorageConverter(IOptions workerOptions, IOptionsSnapshot blobOptions, ILogger logger) { @@ -43,7 +46,6 @@ public async ValueTask ConvertAsync(ConverterContext context) return context?.Source switch { ModelBindingData binding => await ConvertFromBindingDataAsync(context, binding), - CollectionModelBindingData binding => await ConvertFromCollectionBindingDataAsync(context, binding), _ => ConversionResult.Unhandled(), }; } @@ -57,136 +59,191 @@ private async ValueTask ConvertFromBindingDataAsync(ConverterC throw new InvalidBindingSourceException(Constants.BlobExtensionName); } - Dictionary content = GetBindingDataContent(modelBindingData); - var result = await ConvertModelBindingDataAsync(content, context.TargetType, modelBindingData); + BlobBindingData blobData = GetBindingDataContent(modelBindingData); + var result = await ConvertModelBindingDataAsync(context.TargetType, blobData); + + if (result is null) + { + return ConversionResult.Failed(new InvalidOperationException($"Unable to convert blob binding data to type '{context.TargetType.Name}'.")); + } return ConversionResult.Success(result); } + catch (JsonException ex) + { + string msg = string.Format(CultureInfo.CurrentCulture, + @"Binding parameters to complex objects uses JSON serialization. + 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or + 2. Change the blob to be valid json."); + + return ConversionResult.Failed(new InvalidOperationException(msg, ex)); + } catch (Exception ex) { return ConversionResult.Failed(ex); } } - private async ValueTask ConvertFromCollectionBindingDataAsync(ConverterContext context, CollectionModelBindingData collectionModelBindingData) + private BlobBindingData GetBindingDataContent(ModelBindingData bindingData) { - Type elementType = context.TargetType.IsArray - ? context.TargetType.GetElementType() - : context.TargetType.GenericTypeArguments[0]; + if (bindingData is null) + { + throw new ArgumentNullException(nameof(bindingData)); + } - IList result = Array.CreateInstance(elementType, collectionModelBindingData.ModelBindingData.Length); + return bindingData.ContentType switch + { + Constants.JsonContentType => bindingData.Content.ToObjectFromJson(), + _ => throw new InvalidContentTypeException(Constants.JsonContentType) + }; + } - try + private async Task ConvertModelBindingDataAsync(Type targetType, BlobBindingData blobData) + { + if (blobData is null) { - for (var i = 0; i < collectionModelBindingData.ModelBindingData.Length; i++) - { - var modelBindingData = collectionModelBindingData.ModelBindingData[i]; + throw new ArgumentNullException(nameof(blobData)); + } - if (modelBindingData.Source is not Constants.BlobExtensionName) - { - throw new InvalidBindingSourceException(Constants.BlobExtensionName); - } + if (string.IsNullOrEmpty(blobData.Connection)) + { + throw new InvalidOperationException($"'{nameof(blobData.Connection)}' cannot be null or empty."); + } + + if (string.IsNullOrEmpty(blobData.ContainerName)) + { + throw new InvalidOperationException($"'{nameof(blobData.ContainerName)}' cannot be null or empty."); + } - Dictionary content = GetBindingDataContent(modelBindingData); - var element = await ConvertModelBindingDataAsync(content, elementType, modelBindingData); + BlobContainerClient container = CreateBlobContainerClient(blobData.Connection!, blobData.ContainerName!); - if (element is not null) - { - result[i] = element; - } + if (IsCollectionBinding(targetType, blobData.BlobName!, out Type? elementType)) + { + if (elementType is null) + { + throw new InvalidOperationException($"Unable to determine element type for collection binding to type '{targetType.Name}'."); } - if (!context.TargetType.IsArray) + return await BindToCollectionAsync(targetType, elementType, container, blobData.BlobName!); + } + else + { + if (targetType == typeof(BlobContainerClient) && BlobIsFileRegex.IsMatch(blobData.BlobName)) { - var resultType = typeof(List<>).MakeGenericType(elementType); - result = (IList)Activator.CreateInstance(resultType, result); + throw new InvalidOperationException("Binding to a BlobContainerClient with a blob path is not supported. " + + "Either bind to the container path, or use BlobClient instead."); } - return ConversionResult.Success(result); + if (targetType != typeof(BlobContainerClient) && string.IsNullOrEmpty(blobData.BlobName)) + { + throw new InvalidOperationException($"'{nameof(blobData.BlobName)}' cannot be null or empty when binding to a single blob."); + } + + return await ToTargetTypeAsync(targetType, container, blobData.BlobName!); } - catch (JsonException ex) + } + + private bool IsCollectionBinding(Type targetType, string blobName, out Type? elementType) + { + elementType = null; + + // Edge case: These two types should be treated as a single blob binding + // string implements IEnumerable and byte[] would pass the IsArray check + if (targetType == typeof(string) || targetType == typeof(byte[])) { - string msg = String.Format(CultureInfo.CurrentCulture, - @"Binding parameters to complex objects uses JSON serialization. - 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or - 2. Change the blob to be valid json. - The JSON parser failed: {0}", - ex.Message); + return false; + } - return ConversionResult.Failed(new InvalidOperationException(msg, ex)); + if (!(targetType.IsArray || typeof(IEnumerable).IsAssignableFrom(targetType))) + { + return false; } - catch (Exception ex) + + // At this stage, we know we have a collection type binding + elementType = targetType.IsArray ? targetType.GetElementType() : targetType.GenericTypeArguments[0]; + + if (elementType == typeof(BlobContainerClient)) { - return ConversionResult.Failed(ex); + throw new InvalidOperationException("Binding to a BlobContainerClient collection is not supported."); } - } - private Dictionary GetBindingDataContent(ModelBindingData bindingData) - { - return bindingData?.ContentType switch + bool isFile = BlobIsFileRegex.IsMatch(blobName); + + if (isFile && typeof(BlobBaseClient).IsAssignableFrom(elementType)) { - Constants.JsonContentType => new Dictionary(bindingData?.Content?.ToObjectFromJson>(), StringComparer.OrdinalIgnoreCase), - _ => throw new InvalidContentTypeException(Constants.JsonContentType) - }; + throw new InvalidOperationException("Binding to a blob client collection with a blob path is not supported. " + + "Either bind to the container path, or use BlobClient instead."); + } + + // If it's a collection binding and the blob is a file, we should treat it as a single blob binding as it + // end up in the `DeserializeToTargetObjectAsync` method via ToTargetTypeAsync() + if (isFile) + { + return false; + } + + return true; } - private async Task ConvertModelBindingDataAsync(IDictionary content, Type targetType, ModelBindingData bindingData) + private async Task BindToCollectionAsync(Type targetType, Type elementType, BlobContainerClient container, string blobPath) { - content.TryGetValue(Constants.Connection, out var connectionName); - content.TryGetValue(Constants.ContainerName, out var containerName); - content.TryGetValue(Constants.BlobName, out var blobName); + var resultType = typeof(List<>).MakeGenericType(elementType); + var result = (IList)Activator.CreateInstance(resultType); - if (string.IsNullOrEmpty(connectionName)) + await foreach (BlobItem blobItem in container.GetBlobsAsync(prefix: blobPath).ConfigureAwait(false)) { - throw new ArgumentNullException(nameof(connectionName)); + var element = await ToTargetTypeAsync(elementType, container, blobItem.Name); + result.Add(element); } - if (string.IsNullOrEmpty(containerName)) + if (targetType.IsArray) { - throw new ArgumentNullException(nameof(containerName)); + var arrayResult = Array.CreateInstance(elementType, result.Count); + result.CopyTo(arrayResult, 0); + return arrayResult; } - return await ToTargetTypeAsync(targetType, connectionName, containerName, blobName); + return result; } - private async Task ToTargetTypeAsync(Type targetType, string connectionName, string containerName, string blobName) => targetType switch + private async Task ToTargetTypeAsync(Type targetType, BlobContainerClient containerClient, string blobName) => targetType switch { - Type _ when targetType == typeof(string) => await GetBlobStringAsync(connectionName, containerName, blobName), - Type _ when targetType == typeof(Stream) => await GetBlobStreamAsync(connectionName, containerName, blobName), - Type _ when targetType == typeof(byte[]) => await GetBlobBinaryDataAsync(connectionName, containerName, blobName), - Type _ when targetType == typeof(BlobBaseClient) => CreateBlobClient(connectionName, containerName, blobName), - Type _ when targetType == typeof(BlobClient) => CreateBlobClient(connectionName, containerName, blobName), - Type _ when targetType == typeof(BlockBlobClient) => CreateBlobClient(connectionName, containerName, blobName), - Type _ when targetType == typeof(PageBlobClient) => CreateBlobClient(connectionName, containerName, blobName), - Type _ when targetType == typeof(AppendBlobClient) => CreateBlobClient(connectionName, containerName, blobName), - Type _ when targetType == typeof(BlobContainerClient) => CreateBlobContainerClient(connectionName, containerName), - _ => await DeserializeToTargetObjectAsync(targetType, connectionName, containerName, blobName) + Type _ when targetType == typeof(BlobContainerClient) => containerClient, + Type _ when targetType == typeof(string) => await GetBlobStringAsync(containerClient, blobName), + Type _ when targetType == typeof(Stream) => await GetBlobStreamAsync(containerClient, blobName), + Type _ when targetType == typeof(byte[]) => await GetBlobBinaryDataAsync(containerClient, blobName), + Type _ when targetType == typeof(BlobBaseClient) => CreateBlobClient(containerClient, blobName), + Type _ when targetType == typeof(BlobClient) => CreateBlobClient(containerClient, blobName), + Type _ when targetType == typeof(BlockBlobClient) => CreateBlobClient(containerClient, blobName), + Type _ when targetType == typeof(PageBlobClient) => CreateBlobClient(containerClient, blobName), + Type _ when targetType == typeof(AppendBlobClient) => CreateBlobClient(containerClient, blobName), + _ => await DeserializeToTargetObjectAsync(targetType, containerClient, blobName) }; - private async Task DeserializeToTargetObjectAsync(Type targetType, string connectionName, string containerName, string blobName) + private async Task DeserializeToTargetObjectAsync(Type targetType, BlobContainerClient containerClient, string blobName) { - var content = await GetBlobStreamAsync(connectionName, containerName, blobName); + var content = await GetBlobStreamAsync(containerClient, blobName); return _workerOptions?.Value?.Serializer?.Deserialize(content, targetType, CancellationToken.None); } - private async Task GetBlobStringAsync(string connectionName, string containerName, string blobName) + private async Task GetBlobStringAsync(BlobContainerClient containerClient, string blobName) { - var client = CreateBlobClient(connectionName, containerName, blobName); + var client = CreateBlobClient(containerClient, blobName); var download = await client.DownloadContentAsync(); return download.Value.Content.ToString(); } - private async Task GetBlobBinaryDataAsync(string connectionName, string containerName, string blobName) + private async Task GetBlobBinaryDataAsync(BlobContainerClient containerClient, string blobName) { using MemoryStream stream = new(); - var client = CreateBlobClient(connectionName, containerName, blobName); + var client = CreateBlobClient(containerClient, blobName); var res = await client.DownloadToAsync(stream); return stream.ToArray(); } - private async Task GetBlobStreamAsync(string connectionName, string containerName, string blobName) + private async Task GetBlobStreamAsync(BlobContainerClient containerClient, string blobName) { - var client = CreateBlobClient(connectionName, containerName, blobName); + var client = CreateBlobClient(containerClient, blobName); var download = await client.DownloadStreamingAsync(); return download.Value.Content; } @@ -199,26 +256,31 @@ private BlobContainerClient CreateBlobContainerClient(string connectionName, str return container; } - private T CreateBlobClient(string connectionName, string containerName, string blobName) where T : BlobBaseClient + private T CreateBlobClient(BlobContainerClient containerClient, string blobName) where T : BlobBaseClient { if (string.IsNullOrEmpty(blobName)) { throw new ArgumentNullException(nameof(blobName)); } - BlobContainerClient container = CreateBlobContainerClient(connectionName, containerName); - Type targetType = typeof(T); BlobBaseClient blobClient = targetType switch { - Type _ when targetType == typeof(BlobClient) => container.GetBlobClient(blobName), - Type _ when targetType == typeof(BlockBlobClient) => container.GetBlockBlobClient(blobName), - Type _ when targetType == typeof(PageBlobClient) => container.GetPageBlobClient(blobName), - Type _ when targetType == typeof(AppendBlobClient) => container.GetAppendBlobClient(blobName), - _ => container.GetBlobBaseClient(blobName) + Type _ when targetType == typeof(BlobClient) => containerClient.GetBlobClient(blobName), + Type _ when targetType == typeof(BlockBlobClient) => containerClient.GetBlockBlobClient(blobName), + Type _ when targetType == typeof(PageBlobClient) => containerClient.GetPageBlobClient(blobName), + Type _ when targetType == typeof(AppendBlobClient) => containerClient.GetAppendBlobClient(blobName), + _ => containerClient.GetBlobBaseClient(blobName) }; return (T)blobClient; } + + private class BlobBindingData() + { + public string? Connection { get; set; } + public string? ContainerName { get; set; } + public string? BlobName { get; set; } + } } } diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs index 8231635c9..31182c34f 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs @@ -6,8 +6,8 @@ namespace Microsoft.Azure.Functions.Worker { - [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] [InputConverter(typeof(BlobStorageConverter))] + [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] public sealed class BlobTriggerAttribute : TriggerBindingAttribute { private readonly string _blobPath; diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Constants.cs b/extensions/Worker.Extensions.Storage.Blobs/src/Constants.cs index 5320bff21..0766c2a3c 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Constants.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Constants.cs @@ -7,9 +7,6 @@ internal static class Constants { internal const string Storage = "Storage"; internal const string BlobExtensionName = "AzureStorageBlobs"; - internal const string Connection = "Connection"; - internal const string ContainerName = "ContainerName"; - internal const string BlobName = "BlobName"; // Media content types internal const string JsonContentType = "application/json"; diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Properties/AssemblyInfo.cs b/extensions/Worker.Extensions.Storage.Blobs/src/Properties/AssemblyInfo.cs index 7bd16b23e..e8d3e501f 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Properties/AssemblyInfo.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Properties/AssemblyInfo.cs @@ -4,7 +4,7 @@ using System.Runtime.CompilerServices; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; -[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.2")] +[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.3")] [assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Worker.Extensions.Storage.Blobs.csproj b/extensions/Worker.Extensions.Storage.Blobs/src/Worker.Extensions.Storage.Blobs.csproj index ac67a29a9..3685794be 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Worker.Extensions.Storage.Blobs.csproj +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Worker.Extensions.Storage.Blobs.csproj @@ -6,7 +6,7 @@ Azure Blob Storage extensions for .NET isolated functions - 5.1.2 + 6.0.0 false @@ -20,8 +20,8 @@ - - + + diff --git a/samples/WorkerBindingSamples/Blob/BlobInputBindingSamples.cs b/samples/WorkerBindingSamples/Blob/BlobInputBindingSamples.cs index 2f80d49ed..74952c066 100644 --- a/samples/WorkerBindingSamples/Blob/BlobInputBindingSamples.cs +++ b/samples/WorkerBindingSamples/Blob/BlobInputBindingSamples.cs @@ -4,12 +4,16 @@ using System.Net; using System.Text; using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.Extensions.Logging; namespace SampleApp { + /// + /// Samples demonstrating binding to the types supported by the `BlobInput` binding. + /// public class BlobInputBindingSamples { private readonly ILogger _logger; @@ -19,95 +23,116 @@ public BlobInputBindingSamples(ILogger logger) _logger = logger; } - [Function(nameof(BlobInputClientFunction))] - public async Task BlobInputClientFunction( + /// + /// This sample demonstrates how to retrieve the blobs within a container + /// The code uses a instance to read get + /// an async sequence of blobs in the given container. + /// + [Function(nameof(BlobInputContainerClientFunction))] + public async Task BlobInputContainerClientFunction( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("input-container/sample1.txt")] BlobClient client) + [BlobInput("input-container")] BlobContainerClient client) { - var downloadResult = await client.DownloadContentAsync(); - var content = downloadResult.Value.Content.ToString(); + _logger.LogInformation("Blobs within container:"); - _logger.LogInformation("Blob content: {content}", content); + var resultSegment = client.GetBlobsAsync(); + await foreach (BlobItem blob in resultSegment) + { + _logger.LogInformation(blob.Name); + } return req.CreateResponse(HttpStatusCode.OK); } - [Function(nameof(BlobInputStreamFunction))] - public async Task BlobInputStreamFunction( + /// + /// This sample demonstrates how to retrieve the contents of a given blob file. + /// The code uses a instance to read contents of the blob. + /// The instance could also be used for write operations. + /// + [Function(nameof(BlobInputClientFunction))] + public async Task BlobInputClientFunction( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("input-container/sample1.txt")] Stream stream) + [BlobInput("input-container/sample1.txt")] BlobClient client) { - using var blobStreamReader = new StreamReader(stream); - _logger.LogInformation("Blob content: {content}", await blobStreamReader.ReadToEndAsync()); - + var downloadResult = await client.DownloadContentAsync(); + var content = downloadResult.Value.Content.ToString(); + _logger.LogInformation("Blob content: {content}", content); return req.CreateResponse(HttpStatusCode.OK); } - [Function(nameof(BlobInputByteArrayFunction))] - public HttpResponseData BlobInputByteArrayFunction( + /// + /// This sample demonstrates how to retrieve the contents all the blobs within a container. + /// The code uses a of type to retrieve + /// a list of all the blobs within a given container, and then reads + /// the contents of each blob. + /// + [Function(nameof(BlobInputCollectionFunction))] + public async Task BlobInputCollectionFunction( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("input-container/sample1.txt")] Byte[] data) + [BlobInput("input-container")] IEnumerable blobs) { - _logger.LogInformation("Blob content: {content}", Encoding.Default.GetString(data)); - return req.CreateResponse(HttpStatusCode.OK); - } + _logger.LogInformation("Content of all blobs within container:"); - [Function(nameof(BlobInputStringFunction))] - public HttpResponseData BlobInputStringFunction( - [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, string filename, - [BlobInput("input-container/{filename}")] string data) - { - _logger.LogInformation("Blob content: {content}", data); - return req.CreateResponse(HttpStatusCode.OK); - } + foreach (BlobClient blob in blobs) + { + var downloadResult = await blob.DownloadContentAsync(); + var content = downloadResult.Value.Content.ToString(); + _logger.LogInformation(content); + } - [Function(nameof(BlobInputBookFunction))] - public HttpResponseData BlobInputBookFunction( - [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("input-container/book.json")] Book data) - { - _logger.LogInformation("Book name: {name}", data.Name); return req.CreateResponse(HttpStatusCode.OK); } - [Function(nameof(BlobInputCollectionFunction))] - public HttpResponseData BlobInputCollectionFunction( + /// + /// This sample demonstrates how to retrieve the contents of a given blob file + /// by binding to a . This function also demonstrates how to + /// bind to a parameter from the route to get the blob name. + /// Example usage: api/BlobInputStreamFunction?filename=sample1.txt + /// + [Function(nameof(BlobInputStreamFunction))] + public async Task BlobInputStreamFunction( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("input-container", IsBatched = true)] IEnumerable blobs) + [BlobInput("input-container/{filename}")] Stream stream) { - _logger.LogInformation("Blobs within container:"); - - foreach (BlobClient blob in blobs) - { - _logger.LogInformation("Blob name: {blobName}, Container name: {containerName}", blob.Name, blob.BlobContainerName); - } - + using var blobStreamReader = new StreamReader(stream); + _logger.LogInformation("Blob content: {content}", await blobStreamReader.ReadToEndAsync()); return req.CreateResponse(HttpStatusCode.OK); } - [Function(nameof(BlobInputStringArrayFunction))] - public HttpResponseData BlobInputStringArrayFunction( + /// + /// This sample demonstrates how to retrieve the content of all the blobs within a folder in a + /// given container. The code uses a of type + /// to retrieve an array containing all the content of the blobs in the file path. + /// + [Function(nameof(BlobInputCollectionSubdirectoryFunction))] + public HttpResponseData BlobInputCollectionSubdirectoryFunction( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("input-container", IsBatched = true)] string[] blobContent) + [BlobInput("input-container/test")] string[] testFolderBlobs) { - _logger.LogInformation("Content of all blobs within container:"); + _logger.LogInformation("Content of all blobs within the 'test' subdirectory in the container:"); - foreach (var item in blobContent) + foreach (var blobContents in testFolderBlobs) { - _logger.LogInformation(item); + _logger.LogInformation(blobContents); } return req.CreateResponse(HttpStatusCode.OK); } - [Function(nameof(BlobInputBookArrayFunction))] - public HttpResponseData BlobInputBookArrayFunction( + /// + /// This sample demonstrates how to retrieve the contents of a given blob file as a collection. + /// The code uses a of type to retrieve an array containing + /// the contents of the given blob file, which is a JSON array of books. + /// The content of the blob must be JSON deserializable into the type of the parameter. + /// + [Function(nameof(BlobInputBookArrayFileContentFunction))] + public HttpResponseData BlobInputBookArrayFileContentFunction( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("input-container", IsBatched = true)] Book[] books) + [BlobInput("input-container/manybooks.json")] Book[] blobContent) { - _logger.LogInformation("Content of all blobs within container:"); + _logger.LogInformation("Content of single file as array:"); - foreach (var item in books) + foreach (var item in blobContent) { _logger.LogInformation(item.Name); } diff --git a/samples/WorkerBindingSamples/Blob/BlobTriggerBindingSamples.cs b/samples/WorkerBindingSamples/Blob/BlobTriggerBindingSamples.cs index d27cb1c19..b9eebaac3 100644 --- a/samples/WorkerBindingSamples/Blob/BlobTriggerBindingSamples.cs +++ b/samples/WorkerBindingSamples/Blob/BlobTriggerBindingSamples.cs @@ -8,6 +8,9 @@ namespace SampleApp { + /// + /// Samples demonstrating binding to the types supported by the `BlobTrigger` binding. + /// public class BlobTriggerBindingSamples { private readonly ILogger _logger; @@ -17,6 +20,14 @@ public BlobTriggerBindingSamples(ILogger logger) _logger = logger; } + /// + /// This sample demonstrates how to retrieve the contents of a blob file when a blob + /// is added or updated in the given container. The code uses a + /// instance to read contents of the blob. The string {name} in the blob trigger path + /// creates a binding expression that you can use in function code to access the file + /// name of the triggering blob. + /// The instance could also be used for write operations. + /// [Function(nameof(BlobClientFunction))] public async Task BlobClientFunction( [BlobTrigger("client-trigger/{name}")] BlobClient client, string name) @@ -26,6 +37,12 @@ public async Task BlobClientFunction( _logger.LogInformation("Blob name: {name} -- Blob content: {content}", name, content); } + /// + /// This sample demonstrates how to retrieve the contents of a blob file when a blob + /// is added or updated in the given container by binding to a . + /// The string {name} in the blob trigger path creates a binding expression that you + /// can use in function code to access the file name of the triggering blob. + /// [Function(nameof(BlobStreamFunction))] public async Task BlobStreamFunction( [BlobTrigger("stream-trigger/{name}")] Stream stream, string name) @@ -35,6 +52,10 @@ public async Task BlobStreamFunction( _logger.LogInformation("Blob name: {name} -- Blob content: {content}", name, content); } + /// + /// This sample demonstrates how to retrieve the contents of a blob file when a blob + /// is added or updated in the given container by binding to a . + /// [Function(nameof(BlobByteArrayFunction))] public void BlobByteArrayFunction( [BlobTrigger("byte-trigger")] Byte[] data) @@ -42,6 +63,10 @@ public void BlobByteArrayFunction( _logger.LogInformation("Blob content: {content}", Encoding.Default.GetString(data)); } + /// + /// This sample demonstrates how to retrieve the contents of a blob file when a blob + /// is added or updated in the given container by binding to a . + /// [Function(nameof(BlobStringFunction))] public void BlobStringFunction( [BlobTrigger("string-trigger")] string data) @@ -49,6 +74,11 @@ public void BlobStringFunction( _logger.LogInformation("Blob content: {content}", data); } + /// + /// This sample demonstrates how to retrieve the contents of a blob file when a blob + /// is added or updated in the given container by binding to a (POCO). + /// The content of the blob must be JSON deserializable into the type of the parameter. + /// [Function(nameof(BlobBookFunction))] public void BlobBookFunction( [BlobTrigger("book-trigger")] Book data) diff --git a/samples/WorkerBindingSamples/Blob/ExpressionFunction.cs b/samples/WorkerBindingSamples/Blob/ExpressionFunction.cs deleted file mode 100644 index b8be20c0f..000000000 --- a/samples/WorkerBindingSamples/Blob/ExpressionFunction.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using Microsoft.Azure.Functions.Worker; -using Microsoft.Extensions.Logging; - -namespace SampleApp -{ - public class ExpressionFunction - { - private readonly ILogger _logger; - - public ExpressionFunction(ILogger logger) - { - _logger = logger; - } - - [Function(nameof(ExpressionFunction))] - public void Run( - [QueueTrigger("expression-trigger")] Book book, - [BlobInput("input-container/{id}.txt")] string myBlob, - FunctionContext context) - { - _logger.LogInformation("Trigger content: {content}", book); - _logger.LogInformation("Blob content: {content}", myBlob); - } - } -} diff --git a/samples/WorkerBindingSamples/Program.cs b/samples/WorkerBindingSamples/Program.cs index f9c235fa7..e0ff09a1b 100644 --- a/samples/WorkerBindingSamples/Program.cs +++ b/samples/WorkerBindingSamples/Program.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; using Microsoft.Extensions.Hosting; namespace SampleApp @@ -9,6 +10,10 @@ public class Program { public static void Main() { + #if DEBUG + Console.WriteLine($"Azure Functions .NET Worker (PID: { Environment.ProcessId }) initialized in debug mode."); + #endif + var host = new HostBuilder() .ConfigureFunctionsWorkerDefaults() .Build(); diff --git a/samples/WorkerBindingSamples/WorkerBindingSamples.csproj b/samples/WorkerBindingSamples/WorkerBindingSamples.csproj index 7b54a7565..5748826b7 100644 --- a/samples/WorkerBindingSamples/WorkerBindingSamples.csproj +++ b/samples/WorkerBindingSamples/WorkerBindingSamples.csproj @@ -7,13 +7,15 @@ enable - - + - - + + + + + PreserveNewest diff --git a/src/DotNetWorker.Grpc/Properties/AssemblyInfo.cs b/src/DotNetWorker.Grpc/Properties/AssemblyInfo.cs index c11eba4ae..c952293f4 100644 --- a/src/DotNetWorker.Grpc/Properties/AssemblyInfo.cs +++ b/src/DotNetWorker.Grpc/Properties/AssemblyInfo.cs @@ -6,4 +6,4 @@ [assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] [assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] -[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.WorkerExtension.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] +[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] diff --git a/test/DotNetWorkerTests/Properties/AssemblyInfo.cs b/test/DotNetWorkerTests/Properties/AssemblyInfo.cs index 9e991d617..2d7df5e1b 100644 --- a/test/DotNetWorkerTests/Properties/AssemblyInfo.cs +++ b/test/DotNetWorkerTests/Properties/AssemblyInfo.cs @@ -5,5 +5,5 @@ using Xunit; [assembly: CollectionBehavior(DisableTestParallelization = true)] -[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.WorkerExtension.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] +[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] diff --git a/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs b/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs index 3452b7147..63b748ab7 100644 --- a/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs +++ b/test/E2ETests/E2EApps/E2EApp/Blob/BlobInputBindingFunctions.cs @@ -81,7 +81,7 @@ public async Task BlobInputBaseClientTest( [Function(nameof(BlobInputContainerClientTest))] public async Task BlobInputContainerClientTest( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("test-input-dotnet-isolated/testFile.txt")] BlobContainerClient client) + [BlobInput("test-input-dotnet-isolated")] BlobContainerClient client) { var blobClient = client.GetBlobClient("testFile.txt"); var downloadResult = await blobClient.DownloadContentAsync(); @@ -131,17 +131,73 @@ public async Task BlobInputPocoTest( return response; } - [Function(nameof(BlobInputCollectionTest))] - public async Task BlobInputCollectionTest( + [Function(nameof(BlobInputClientArrayTest))] + public async Task BlobInputClientArrayTest( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("test-input-dotnet-isolated", IsBatched = true)] IEnumerable blobs) + [BlobInput("test-input-dotnet-isolated")] BlobClient[] blobs) { List blobList = new(); foreach (BlobClient blob in blobs) { - _logger.LogInformation("Blob name: {blobName}, Container name: {containerName}", blob.Name, blob.BlobContainerName); - blobList.Add(blob.Name); + var downloadResult = await blob.DownloadContentAsync(); + var content = downloadResult.Value.Content.ToString(); + blobList.Add(content); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobList.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputStreamArrayTest))] + public async Task BlobInputStreamArrayTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated")] Stream[] blobContent) + { + List blobList = new(); + + foreach (Stream stream in blobContent) + { + using var blobStreamReader = new StreamReader(stream); + blobList.Add(blobStreamReader.ReadToEnd()); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobList.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputBytesArrayTest))] + public async Task BlobInputBytesArrayTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated")] byte[][] blobContent) + { + List blobList = new(); + + foreach (byte[] bytes in blobContent) + { + blobList.Add(Encoding.Default.GetString(bytes)); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobList.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputBytesArraySingleBlobTest))] + public async Task BlobInputBytesArraySingleBlobTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] byte[][] blobContent) + { + List blobList = new(); + + foreach (byte[] bytes in blobContent) + { + blobList.Add(Encoding.Default.GetString(bytes)); } var response = req.CreateResponse(HttpStatusCode.OK); @@ -153,7 +209,18 @@ public async Task BlobInputCollectionTest( [Function(nameof(BlobInputStringArrayTest))] public async Task BlobInputStringArrayTest( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("test-input-dotnet-isolated", IsBatched = true)] string[] blobContent) + [BlobInput("test-input-dotnet-isolated")] string[] blobContent) + { + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobContent); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputStringArraySingleBlobTest))] + public async Task BlobInputStringArraySingleBlobTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] string[] blobContent) { var response = req.CreateResponse(HttpStatusCode.OK); string contentAsString = string.Join(", ", blobContent); @@ -164,7 +231,25 @@ public async Task BlobInputStringArrayTest( [Function(nameof(BlobInputPocoArrayTest))] public async Task BlobInputPocoArrayTest( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [BlobInput("test-input-dotnet-isolated", IsBatched = true)] Book[] books) + [BlobInput("test-input-dotnet-isolated")] Book[] books) + { + List bookNames = new(); + + foreach (var item in books) + { + bookNames.Add(item.Name); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", bookNames.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputPocoArraySingleBlobTest))] + public async Task BlobInputPocoArraySingleBlobTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] Book[] books) { List bookNames = new(); @@ -178,5 +263,158 @@ public async Task BlobInputPocoArrayTest( await response.WriteStringAsync(contentAsString); return response; } + + [Function(nameof(BlobInputClientEnumerableTest))] + public async Task BlobInputClientEnumerableTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated")] IEnumerable blobs) + { + List blobList = new(); + + foreach (BlobClient blob in blobs) + { + var downloadResult = await blob.DownloadContentAsync(); + var content = downloadResult.Value.Content.ToString(); + blobList.Add(content); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobList.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputStreamEnumerableTest))] + public async Task BlobInputStreamEnumerableTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated")] IEnumerable blobContent) + { + List blobList = new(); + + foreach (Stream stream in blobContent) + { + using var blobStreamReader = new StreamReader(stream); + blobList.Add(blobStreamReader.ReadToEnd()); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobList.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputBytesEnumerableTest))] + public async Task BlobInputBytesEnumerableTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated")] IEnumerable blobContent) + { + List blobList = new(); + + foreach (byte[] bytes in blobContent) + { + blobList.Add(Encoding.Default.GetString(bytes)); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobList.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputBytesEnumerableSingleBlobTest))] + public async Task BlobInputBytesEnumerableSingleBlobTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] IEnumerable blobContent) + { + List blobList = new(); + + foreach (byte[] bytes in blobContent) + { + blobList.Add(Encoding.Default.GetString(bytes)); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobList.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputStringEnumerableTest))] + public async Task BlobInputStringEnumerableTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated")] IEnumerable blobContent) + { + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobContent); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputStringEnumerableSingleBlobTest))] + public async Task BlobInputStringEnumerableSingleBlobTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] IEnumerable blobContent) + { + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobContent); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputPocoEnumerableTest))] + public async Task BlobInputPocoEnumerableTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated")] IEnumerable books) + { + List bookNames = new(); + + foreach (var item in books) + { + bookNames.Add(item.Name); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", bookNames.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputPocoEnumerableSingleBlobTest))] + public async Task BlobInputPocoEnumerableSingleBlobTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/testFile.txt")] IEnumerable books) + { + List bookNames = new(); + + foreach (var item in books) + { + bookNames.Add(item.Name); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", bookNames.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } + + [Function(nameof(BlobInputClientCollectionWithSubdirectoryTest))] + public async Task BlobInputClientCollectionWithSubdirectoryTest( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [BlobInput("test-input-dotnet-isolated/test")] IEnumerable blobs) + { + List blobList = new(); + + foreach (BlobClient blob in blobs) + { + var downloadResult = await blob.DownloadContentAsync(); + var content = downloadResult.Value.Content.ToString(); + blobList.Add(content); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + string contentAsString = string.Join(", ", blobList.ToArray()); + await response.WriteStringAsync(contentAsString); + return response; + } } } \ No newline at end of file diff --git a/test/E2ETests/E2EApps/E2EApp/Blob/BlobTriggerBindingFunctions.cs b/test/E2ETests/E2EApps/E2EApp/Blob/BlobTriggerBindingFunctions.cs index ecce43f1a..a992cbbac 100644 --- a/test/E2ETests/E2EApps/E2EApp/Blob/BlobTriggerBindingFunctions.cs +++ b/test/E2ETests/E2EApps/E2EApp/Blob/BlobTriggerBindingFunctions.cs @@ -70,17 +70,6 @@ public async Task BlobTriggerBlobClientTest( _logger.LogInformation("BlobClientTriggerOutput: {c}", content); } - [Function(nameof(BlobTriggerBlobContainerClientTest))] - public async Task BlobTriggerBlobContainerClientTest( - [BlobTrigger("test-trigger-containerclient-dotnet-isolated/{name}")] BlobContainerClient client, string name, - FunctionContext context) - { - var blobClient = client.GetBlobClient(name); - var downloadResult = await blobClient.DownloadContentAsync(); - string content = downloadResult.Value.Content.ToString(); - _logger.LogInformation("BlobContainerTriggerOutput: {c}", content); - } - public class TestBlobData { [JsonPropertyName("text")] diff --git a/test/E2ETests/E2ETests/E2ETests.csproj b/test/E2ETests/E2ETests/E2ETests.csproj index 65eb13ef8..7390bdcba 100644 --- a/test/E2ETests/E2ETests/E2ETests.csproj +++ b/test/E2ETests/E2ETests/E2ETests.csproj @@ -8,7 +8,7 @@ - + diff --git a/test/E2ETests/E2ETests/Helpers/StorageHelpers.cs b/test/E2ETests/E2ETests/Helpers/StorageHelpers.cs index f710ebcd7..b3170aa03 100644 --- a/test/E2ETests/E2ETests/Helpers/StorageHelpers.cs +++ b/test/E2ETests/E2ETests/Helpers/StorageHelpers.cs @@ -121,9 +121,19 @@ public static Task UploadFileToContainer(string containerName, string fileName) return UploadFileToContainer(containerName, fileName, "Hello World"); } - public async static Task UploadFileToContainer(string containerName, string fileName, string fileContents) + public async static Task UploadFileToContainer(string containerName, string fileName, string fileContents, bool containsSubdirectory = false) { string sourceFile = $"{fileName}.txt"; + + if (containsSubdirectory) + { + string directoryPath = Path.GetDirectoryName(sourceFile); + if (!Directory.Exists(directoryPath)) + { + Directory.CreateDirectory(directoryPath); + } + } + File.WriteAllText(sourceFile, fileContents); await CreateBlobContainer(containerName); BlobContainerClient cloudBlobContainer = CreateBlobContainerClient(containerName); diff --git a/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs b/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs index 708c7d04c..ab815f147 100644 --- a/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs +++ b/test/E2ETests/E2ETests/Storage/BlobEndToEndTests.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Text; using System.Text.Json; using System.Threading.Tasks; using Xunit; @@ -134,33 +135,6 @@ await TestUtility.RetryAsync(() => Assert.Equal("Hello World", result); } - [Fact] - public async Task BlobTrigger_BlobContainerClient_Succeeds() - { - string key = "BlobContainerTriggerOutput: "; - string fileName = Guid.NewGuid().ToString(); - - //Cleanup - await StorageHelpers.ClearBlobContainers(); - - //Trigger - await StorageHelpers.UploadFileToContainer(Constants.Blob.TriggerBlobContainerClientContainer, fileName); - - //Verify - IEnumerable logs = null; - await TestUtility.RetryAsync(() => - { - logs = _fixture.TestLogs.CoreToolsLogs.Where(p => p.Contains(key)); - return Task.FromResult(logs.Count() >= 1); - }); - - var lastLog = logs.Last(); - int subStringStart = lastLog.LastIndexOf(key) + key.Length; - var result = lastLog[subStringStart..]; - - Assert.Equal("Hello World", result); - } - [Theory] [InlineData("BlobInputClientTest")] [InlineData("BlobInputBlockClientTest")] @@ -214,22 +188,30 @@ public async Task BlobInput_Poco_Succeeds() Assert.Contains(expectedMessage, actualMessage); } - [Fact] - public async Task BlobInput_BlobClientCollection_Succeeds() + [Theory] + [InlineData("BlobInputClientArrayTest")] + [InlineData("BlobInputClientEnumerableTest")] + [InlineData("BlobInputStreamArrayTest")] + [InlineData("BlobInputStreamEnumerableTest")] + [InlineData("BlobInputBytesArrayTest")] + [InlineData("BlobInputBytesEnumerableTest")] + [InlineData("BlobInputStringArrayTest")] + [InlineData("BlobInputStringEnumerableTest")] + public async Task BlobInput_BlobCollection_Succeeds(string functionName) { - string expectedMessage = "testFile1.txt, testFile2.txt, testFile3.txt"; + string expectedMessage = "ABC, DEF, GHI"; HttpStatusCode expectedStatusCode = HttpStatusCode.OK; //Cleanup await StorageHelpers.ClearBlobContainers(); //Setup - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile1"); - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile2"); - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile3"); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile1", "ABC"); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile2", "DEF"); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile3", "GHI"); //Trigger - HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputCollectionTest"); + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionName); string actualMessage = await response.Content.ReadAsStringAsync(); //Verify @@ -237,22 +219,29 @@ public async Task BlobInput_BlobClientCollection_Succeeds() Assert.Equal(expectedMessage, actualMessage); } - [Fact] - public async Task BlobInput_StringCollection_Succeeds() + [Theory] + [InlineData("BlobInputPocoArrayTest")] + [InlineData("BlobInputPocoEnumerableTest")] + public async Task BlobInput_PocoCollection_Succeeds(string functionName) { - string expectedMessage = "ABC, DEF, GHI"; + string book1 = $@"{{ ""id"": ""1"", ""name"": ""To Kill a Mockingbird""}}"; + string book2 = $@"{{ ""id"": ""2"", ""name"": ""Of Mice and Men""}}"; + string book3 = $@"{{ ""id"": ""3"", ""name"": ""The Wind in the Willows""}}"; + + string expectedMessage = "To Kill a Mockingbird, Of Mice and Men, The Wind in the Willows"; HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + //Cleanup await StorageHelpers.ClearBlobContainers(); //Setup - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile1", "ABC"); - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile2", "DEF"); - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile3", "GHI"); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "book1", book1); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "book2", book2); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "book3", book3); //Trigger - HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputStringArrayTest"); + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionName); string actualMessage = await response.Content.ReadAsStringAsync(); //Verify @@ -260,27 +249,94 @@ public async Task BlobInput_StringCollection_Succeeds() Assert.Equal(expectedMessage, actualMessage); } - [Fact] - public async Task BlobInput_PocoCollection_Succeeds() + [Theory] + [InlineData("BlobInputStringArraySingleBlobTest")] + [InlineData("BlobInputStringEnumerableSingleBlobTest")] + public async Task BlobInput_StringCollection_WithSingleBlobFile_Succeeds(string functionName) { - string book1 = $@"{{ ""id"": ""1"", ""name"": ""To Kill a Mockingbird""}}"; - string book2 = $@"{{ ""id"": ""2"", ""name"": ""Of Mice and Men""}}"; - string book3 = $@"{{ ""id"": ""3"", ""name"": ""The Wind in the Willows""}}"; + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; - string expectedMessage = "To Kill a Mockingbird, Of Mice and Men, The Wind in the Willows"; + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Setup + var fileContent = JsonSerializer.Serialize(new string[] { "Hello", "World" }); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile", fileContent); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionName); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Equal("Hello, World", actualMessage); + } + + [Theory] + [InlineData("BlobInputBytesArraySingleBlobTest")] + [InlineData("BlobInputBytesEnumerableSingleBlobTest")] + public async Task BlobInput_ByteArrayCollection_WithSingleBlobFile_Succeeds(string functionName) + { HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Setup + var data = new List { Encoding.UTF8.GetBytes("Item1"), Encoding.UTF8.GetBytes("Item2") }; + var fileContent = JsonSerializer.Serialize(data); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile", fileContent); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionName); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Equal("Item1, Item2", actualMessage); + } + + + [Theory] + [InlineData("BlobInputPocoArraySingleBlobTest")] + [InlineData("BlobInputPocoEnumerableSingleBlobTest")] + public async Task BlobInput_PocoCollection_WithSingleBlobFile_Succeeds(string functionName) + { + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; //Cleanup await StorageHelpers.ClearBlobContainers(); //Setup - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "book1", book1); - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "book2", book2); - await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "book3", book3); + var data = new List { new { id = "1", name = "To Kill a Mockingbird" }, new { id = "2", name = "Of Mice and Men" } }; + var fileContent = JsonSerializer.Serialize(data); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "testFile", fileContent); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionName); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Equal("To Kill a Mockingbird, Of Mice and Men", actualMessage); + } + + [Fact] + public async Task BlobInput_BlobCollection_WithSubdirectory_Succeeds() + { + string expectedMessage = "ABC, GHI"; + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + //Cleanup + await StorageHelpers.ClearBlobContainers(); + + //Setup + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "test/file1", "ABC", true); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "file2", "DEF"); + await StorageHelpers.UploadFileToContainer(Constants.Blob.InputBindingContainer, "test/file3", "GHI", true); //Trigger - HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputPocoArrayTest"); + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("BlobInputClientCollectionWithSubdirectoryTest"); string actualMessage = await response.Content.ReadAsStringAsync(); //Verify diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 2dd9dad39..3189d7dea 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -171,7 +171,7 @@ public void StorageFunctions() AssertDictionary(extensions, new Dictionary { - { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.3" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.2" }, { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.3" }, }); @@ -208,10 +208,8 @@ void ValidateBlobInput(ExpandoObject b) { { "Name", "blobinput" }, { "Type", "blob" }, - { "DataType", "String"}, { "Direction", "In" }, { "blobPath", "container2" }, - { "Cardinality", "Many" }, { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } }); } @@ -227,7 +225,6 @@ void ValidateBlobTrigger(ExpandoObject b) { "Name", "blob" }, { "Type", "blobTrigger" }, { "Direction", "In" }, - { "DataType", "String"}, { "path", "container2/%file%" }, { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } }); @@ -321,7 +318,6 @@ void ValidateBlobInput(ExpandoObject b) { "Type", "blob" }, { "Direction", "In" }, { "blobPath", "container2/%file%" }, - { "Cardinality", "One" }, { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } }); } @@ -371,7 +367,6 @@ void ValidateBlobInputForEnumerable(ExpandoObject b) { "Type", "blob" }, { "Direction", "In" }, { "blobPath", "container2" }, - { "Cardinality", "Many" }, { "Properties", new Dictionary( ) { { "SupportsDeferredBinding", "True" } } } }); } @@ -391,8 +386,6 @@ void ValidateBlobInputForStringArray(ExpandoObject b) { "Type", "blob" }, { "Direction", "In" }, { "blobPath", "container2" }, - { "Cardinality", "Many" }, - { "DataType", "String" }, { "Properties", new Dictionary( ) { { "SupportsDeferredBinding", "True" } } } }); } @@ -487,7 +480,7 @@ public void MultiOutput_OnReturnType() AssertDictionary(extensions, new Dictionary { - { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.3" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.2" }, { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.3" }, }); @@ -939,7 +932,7 @@ public object BlobToQueue( [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] public object BlobToBlobs( [BlobTrigger("container2/%file%")] string blob, - [BlobInput("container2", IsBatched = true)] IEnumerable blobinput) + [BlobInput("container2")] IEnumerable blobinput) { throw new NotImplementedException(); } @@ -956,7 +949,6 @@ public object BlobStringToBlobStringFunction( throw new NotImplementedException(); } - [Function("BlobClientToBlobStringFunction")] [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] public object BlobClientToBlobStreamFunction( @@ -1000,7 +992,7 @@ private class SDKTypeBindings_BlobCollection [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] public object BlobStringToBlobStringArray( [BlobTrigger("container2/%file%")] string blob, - [BlobInput("container2", IsBatched = true)] string[] blobinput) + [BlobInput("container2")] string[] blobinput) { throw new NotImplementedException(); } @@ -1010,7 +1002,7 @@ public object BlobStringToBlobStringArray( [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] public object BlobStringToBlobClientEnumerable( [BlobTrigger("container2/%file%")] string blob, - [BlobInput("container2", IsBatched = true)] IEnumerable blobinput) + [BlobInput("container2")] IEnumerable blobinput) { throw new NotImplementedException(); } @@ -1019,7 +1011,7 @@ public object BlobStringToBlobClientEnumerable( [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] public object BlobStringToBlobPocoEnumerable( [BlobTrigger("container2/%file%")] string blob, - [BlobInput("container2", IsBatched = true)] IEnumerable blobinput) + [BlobInput("container2")] IEnumerable blobinput) { throw new NotImplementedException(); } @@ -1028,7 +1020,7 @@ public object BlobStringToBlobPocoEnumerable( [BlobOutput("container1/hello.txt", Connection = "MyOtherConnection")] public object BlobStringToBlobPocoArray( [BlobTrigger("container2/%file%")] string blob, - [BlobInput("container2", IsBatched = true)] Poco[] blobinput) + [BlobInput("container2")] Poco[] blobinput) { throw new NotImplementedException(); } diff --git a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs index bf2b86883..513302dd6 100644 --- a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs +++ b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/IntegratedTriggersAndBindingsTests.cs @@ -213,8 +213,8 @@ public Task> GetFunctionMetadataAsync(string d { var metadataList = new List(); var Function0RawBindings = new List(); - Function0RawBindings.Add(@"{""name"":""req"",""type"":""HttpTrigger"",""direction"":""In"",""methods"":[""get"",""post""]}"); - Function0RawBindings.Add(@"{""name"":""myBlob"",""type"":""Blob"",""direction"":""In"",""blobPath"":""test-samples/sample1.txt"",""connection"":""AzureWebJobsStorage"",""cardinality"":""One"",""dataType"":""String""}"); + Function0RawBindings.Add(@"{""name"":""req"",""type"":""HttpTrigger"",""direction"":""In"",""authLevel"":""Anonymous"",""methods"":[""get"",""post""]}"); + Function0RawBindings.Add(@"{""name"":""myBlob"",""type"":""Blob"",""direction"":""In"",""properties"":{""supportsDeferredBinding"":""True""},""blobPath"":""test-samples/sample1.txt"",""connection"":""AzureWebJobsStorage"",""dataType"":""String""}"); Function0RawBindings.Add(@"{""name"":""Book"",""type"":""Queue"",""direction"":""Out"",""queueName"":""functionstesting2"",""connection"":""AzureWebJobsStorage""}"); Function0RawBindings.Add(@"{""name"":""HttpResponse"",""type"":""http"",""direction"":""Out""}"); diff --git a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs index c66ac9058..59f37ec6d 100644 --- a/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs +++ b/test/Sdk.Generator.Tests/FunctionMetadataProviderGeneratorTests/StorageBindingTests.cs @@ -164,7 +164,7 @@ public object BlobToQueue( [Function("BlobsToQueueFunction")] [QueueOutput("queue2")] public object BlobsToQueue( - [BlobInput("container2", IsBatched = true)] IEnumerable blobs) + [BlobInput("container2")] IEnumerable blobs) { throw new NotImplementedException(); } @@ -219,7 +219,7 @@ public Task> GetFunctionMetadataAsync(string d metadataList.Add(Function1); var Function2RawBindings = new List(); Function2RawBindings.Add(@"{""name"":""$return"",""type"":""Queue"",""direction"":""Out"",""queueName"":""queue2""}"); - Function2RawBindings.Add(@"{""name"":""blobs"",""type"":""Blob"",""direction"":""In"",""blobPath"":""container2"",""cardinality"":""Many""}"); + Function2RawBindings.Add(@"{""name"":""blobs"",""type"":""Blob"",""direction"":""In"",""properties"":{""supportsDeferredBinding"":""True""},""blobPath"":""container2""}"); var Function2 = new DefaultFunctionMetadata { @@ -259,51 +259,6 @@ await TestHelpers.RunTestAsync( expectedGeneratedFileName, expectedOutput); } - - [Fact] - public async void TestInvalidBlobCardinalityMany() - { - string inputCode = """ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Net; - using System.Text.Json.Serialization; - using Microsoft.Azure.Functions.Worker; - using Microsoft.Azure.Functions.Worker.Http; - - namespace FunctionApp - { - public class BlobTest - { - [Function("Function1")] - public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, - [BlobInput("input-container", Connection = "AzureWebJobsStorage", IsBatched = true)] string blobs) - { - throw new NotImplementedException(); - } - } - } - """; - - string? expectedGeneratedFileName = null; - string? expectedOutput = null; - - var expectedDiagnosticResults = new List - { - new DiagnosticResult(DiagnosticDescriptors.InvalidCardinality) - .WithSpan(15, 105, 15, 110) - // these arguments are the values we pass as the message format parameters when creating the DiagnosticDescriptor instance. - .WithArguments("blobs") - }; - - await TestHelpers.RunTestAsync( - _referencedExtensionAssemblies, - inputCode, - expectedGeneratedFileName, - expectedOutput, - expectedDiagnosticResults); - } } } } diff --git a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata index 2d151241b..87adc894c 100644 --- a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata +++ b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata @@ -1,8 +1,8 @@ [ { - "name": "BlobInputClientFunction", + "name": "BlobInputContainerClientFunction", "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputClientFunction", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputContainerClientFunction", "language": "dotnet-isolated", "properties": { "IsCodeless": false @@ -23,8 +23,7 @@ "name": "client", "direction": "In", "type": "blob", - "blobPath": "input-container/sample1.txt", - "cardinality": "One", + "blobPath": "input-container", "properties": { "supportsDeferredBinding": "True" } @@ -37,9 +36,9 @@ ] }, { - "name": "BlobInputStreamFunction", + "name": "BlobInputClientFunction", "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputStreamFunction", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputClientFunction", "language": "dotnet-isolated", "properties": { "IsCodeless": false @@ -57,11 +56,10 @@ "properties": {} }, { - "name": "stream", + "name": "client", "direction": "In", "type": "blob", "blobPath": "input-container/sample1.txt", - "cardinality": "One", "properties": { "supportsDeferredBinding": "True" } @@ -74,9 +72,9 @@ ] }, { - "name": "BlobInputByteArrayFunction", + "name": "BlobInputCollectionFunction", "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputByteArrayFunction", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputCollectionFunction", "language": "dotnet-isolated", "properties": { "IsCodeless": false @@ -94,11 +92,10 @@ "properties": {} }, { - "name": "data", + "name": "blobs", "direction": "In", "type": "blob", - "blobPath": "input-container/sample1.txt", - "cardinality": "One", + "blobPath": "input-container", "properties": { "supportsDeferredBinding": "True" } @@ -111,9 +108,9 @@ ] }, { - "name": "BlobInputStringFunction", + "name": "BlobInputStreamFunction", "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputStringFunction", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputStreamFunction", "language": "dotnet-isolated", "properties": { "IsCodeless": false @@ -131,48 +128,10 @@ "properties": {} }, { - "name": "data", + "name": "stream", "direction": "In", "type": "blob", "blobPath": "input-container/{filename}", - "cardinality": "One", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "BlobInputBookFunction", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputBookFunction", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "properties": {} - }, - { - "name": "data", - "direction": "In", - "type": "blob", - "blobPath": "input-container/book.json", - "cardinality": "One", "properties": { "supportsDeferredBinding": "True" } @@ -185,9 +144,9 @@ ] }, { - "name": "BlobInputCollectionFunction", + "name": "BlobInputCollectionSubdirectoryFunction", "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputCollectionFunction", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputCollectionSubdirectoryFunction", "language": "dotnet-isolated", "properties": { "IsCodeless": false @@ -205,11 +164,10 @@ "properties": {} }, { - "name": "blobs", + "name": "testFolderBlobs", "direction": "In", "type": "blob", - "blobPath": "input-container", - "cardinality": "Many", + "blobPath": "input-container/test", "properties": { "supportsDeferredBinding": "True" } @@ -222,9 +180,9 @@ ] }, { - "name": "BlobInputStringArrayFunction", + "name": "BlobInputBookArrayFileContentFunction", "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputStringArrayFunction", + "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputBookArrayFileContentFunction", "language": "dotnet-isolated", "properties": { "IsCodeless": false @@ -245,46 +203,7 @@ "name": "blobContent", "direction": "In", "type": "blob", - "blobPath": "input-container", - "cardinality": "Many", - "dataType": "String", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "BlobInputBookArrayFunction", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.BlobInputBindingSamples.BlobInputBookArrayFunction", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "properties": {} - }, - { - "name": "books", - "direction": "In", - "type": "blob", - "blobPath": "input-container", - "cardinality": "Many", + "blobPath": "input-container/manybooks.json", "properties": { "supportsDeferredBinding": "True" } @@ -395,398 +314,5 @@ } } ] - }, - { - "name": "ExpressionFunction", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.ExpressionFunction.Run", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "book", - "direction": "In", - "type": "queueTrigger", - "queueName": "expression-trigger", - "properties": {} - }, - { - "name": "myBlob", - "direction": "In", - "type": "blob", - "blobPath": "input-container/{id}.txt", - "cardinality": "One", - "properties": { - "supportsDeferredBinding": "True" - } - } - ] - }, - { - "name": "DocsByUsingCosmosClient", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsByUsingCosmosClient", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "properties": {} - }, - { - "name": "client", - "direction": "In", - "type": "cosmosDB", - "databaseName": "", - "containerName": "", - "connection": "CosmosDBConnection", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "DocsByUsingDatabaseClient", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsByUsingDatabaseClient", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "properties": {} - }, - { - "name": "database", - "direction": "In", - "type": "cosmosDB", - "databaseName": "ToDoItems", - "containerName": "", - "connection": "CosmosDBConnection", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "DocsByUsingContainerClient", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsByUsingContainerClient", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "properties": {} - }, - { - "name": "container", - "direction": "In", - "type": "cosmosDB", - "databaseName": "ToDoItems", - "containerName": "Items", - "connection": "CosmosDBConnection", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "DocByIdFromQueryString", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromQueryString", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "properties": {} - }, - { - "name": "toDoItem", - "direction": "In", - "type": "cosmosDB", - "databaseName": "ToDoItems", - "containerName": "Items", - "connection": "CosmosDBConnection", - "id": "{Query.id}", - "partitionKey": "{Query.partitionKey}", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "DocByIdFromRouteData", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromRouteData", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "route": "todoitems/{partitionKey}/{id}", - "properties": {} - }, - { - "name": "toDoItem", - "direction": "In", - "type": "cosmosDB", - "databaseName": "ToDoItems", - "containerName": "Items", - "connection": "CosmosDBConnection", - "id": "{id}", - "partitionKey": "{partitionKey}", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "DocByIdFromRouteDataUsingSqlQuery", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromRouteDataUsingSqlQuery", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "route": "todoitems2/{id}", - "properties": {} - }, - { - "name": "toDoItems", - "direction": "In", - "type": "cosmosDB", - "databaseName": "ToDoItems", - "containerName": "Items", - "connection": "CosmosDBConnection", - "sqlQuery": "SELECT * FROM ToDoItems t where t.id = {id}", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "DocByIdFromQueryStringUsingSqlQuery", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromQueryStringUsingSqlQuery", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "properties": {} - }, - { - "name": "toDoItems", - "direction": "In", - "type": "cosmosDB", - "databaseName": "ToDoItems", - "containerName": "Items", - "connection": "CosmosDBConnection", - "sqlQuery": "SELECT * FROM ToDoItems t where t.id = {id}", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "DocsBySqlQuery", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsBySqlQuery", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "req", - "direction": "In", - "type": "httpTrigger", - "authLevel": "Function", - "methods": [ - "get", - "post" - ], - "properties": {} - }, - { - "name": "toDoItems", - "direction": "In", - "type": "cosmosDB", - "databaseName": "ToDoItems", - "containerName": "Items", - "connection": "CosmosDBConnection", - "sqlQuery": "SELECT * FROM ToDoItems t WHERE CONTAINS(t.description, 'cat')", - "properties": { - "supportsDeferredBinding": "True" - } - }, - { - "name": "$return", - "type": "http", - "direction": "Out" - } - ] - }, - { - "name": "DocByIdFromJSON", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromJSON", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "toDoItemLookup", - "direction": "In", - "type": "queueTrigger", - "queueName": "todoqueueforlookup", - "properties": {} - }, - { - "name": "toDoItem", - "direction": "In", - "type": "cosmosDB", - "databaseName": "ToDoItems", - "containerName": "Items", - "connection": "CosmosDBConnection", - "id": "{ToDoItemId}", - "partitionKey": "{ToDoItemPartitionKeyValue}", - "properties": { - "supportsDeferredBinding": "True" - } - } - ] - }, - { - "name": "CosmosTriggerFunction", - "scriptFile": "WorkerBindingSamples.dll", - "entryPoint": "SampleApp.CosmosTriggerFunction.Run", - "language": "dotnet-isolated", - "properties": { - "IsCodeless": false - }, - "bindings": [ - { - "name": "todoItems", - "direction": "In", - "type": "cosmosDBTrigger", - "databaseName": "ToDoItems", - "containerName": "TriggerItems", - "connection": "CosmosDBConnection", - "createLeaseContainerIfNotExists": true, - "properties": {} - } - ] } ] \ No newline at end of file diff --git a/test/SdkE2ETests/Contents/functions.metadata b/test/SdkE2ETests/Contents/functions.metadata index 2f4d1173f..72461af80 100644 --- a/test/SdkE2ETests/Contents/functions.metadata +++ b/test/SdkE2ETests/Contents/functions.metadata @@ -52,7 +52,6 @@ "type": "blob", "blobPath": "test-samples/sample1.txt", "connection": "AzureWebJobsStorage", - "cardinality": "One", "properties": { "supportsDeferredBinding": "True" } @@ -185,7 +184,6 @@ "type": "blob", "blobPath": "test-samples/sample1.txt", "connection": "AzureWebJobsStorage", - "cardinality": "One", "properties": { "supportsDeferredBinding": "True" } diff --git a/test/Worker.Extensions.Tests/Blob/BlobClientTests.cs b/test/Worker.Extensions.Tests/Blob/BlobClientTests.cs new file mode 100644 index 000000000..b1626e7a9 --- /dev/null +++ b/test/Worker.Extensions.Tests/Blob/BlobClientTests.cs @@ -0,0 +1,222 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +// Scenarios for BlobBaseClient, BlockBlobClient, PageBlobClient, and AppendBlobClient +// are tested via E2E tests as Moq does not support mocking extension methods directly +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Blob +{ + public class BlobClientTests + { + private readonly BlobStorageConverter _blobStorageConverter; + private readonly Mock _mockBlobServiceClient; + + public BlobClientTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + + var workerOptions = host.Services.GetService>(); + var logger = host.Services.GetService>(); + + _mockBlobServiceClient = new Mock(); + + var mockBlobOptions = new Mock(); + mockBlobOptions.Object.Client = _mockBlobServiceClient.Object; + + var mockBlobOptionsSnapshot = new Mock>(); + mockBlobOptionsSnapshot + .Setup(m => m.Get(It.IsAny())) + .Returns(mockBlobOptions.Object); + + _blobStorageConverter = new BlobStorageConverter(workerOptions, mockBlobOptionsSnapshot.Object, logger); + } + + [Fact] + public async Task ConvertAsync_BlobClient_WithFilePath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("MyBlob", clientResult.Name); + } + + [Fact] + public async Task ConvertAsync_BlobClient_FilePathWithoutFileExtension_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "test"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.Name).Returns("test"); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("test", clientResult.Name); + } + + [Fact] + public async Task ConvertAsync_BlobClient_WithContainerPath_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("'BlobName' cannot be null or empty when binding to a single blob.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_BlobClientCollection_List_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); + + var mockResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (IEnumerable)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(clientResult); + Assert.Equal("MyBlob", clientResult.First().Name); + } + + [Fact] + public async Task ConvertAsync_BlobClientCollection_Array_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient[]), grpcModelBindingData); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); + + var mockResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(clientResult); + Assert.Equal("MyBlob", clientResult.First().Name); + } + + [Fact] + public async Task ConvertAsync_BlobClientCollection_WithSubdirectoryPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData("test"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient[]), grpcModelBindingData); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); + + var mockResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(clientResult); + Assert.Equal("MyBlob", clientResult.First().Name); + } + + [Fact] + public async Task ConvertAsync_BlobClientCollection_WithFilePath_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var mockContainer = new Mock(); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Contains("Binding to a blob client collection with a blob path is not supported.", conversionResult.Error.Message); + } + } +} diff --git a/test/Worker.Extensions.Tests/Blob/BlobContainerClientTests.cs b/test/Worker.Extensions.Tests/Blob/BlobContainerClientTests.cs new file mode 100644 index 000000000..913ffebf4 --- /dev/null +++ b/test/Worker.Extensions.Tests/Blob/BlobContainerClientTests.cs @@ -0,0 +1,149 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Blob +{ + public class BlobContainerClientTests + { + private readonly BlobStorageConverter _blobStorageConverter; + private readonly Mock _mockBlobServiceClient; + + public BlobContainerClientTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + + var workerOptions = host.Services.GetService>(); + var logger = host.Services.GetService>(); + + _mockBlobServiceClient = new Mock(); + + var mockBlobOptions = new Mock(); + mockBlobOptions.Object.Client = _mockBlobServiceClient.Object; + + var mockBlobOptionsSnapshot = new Mock>(); + mockBlobOptionsSnapshot + .Setup(m => m.Get(It.IsAny())) + .Returns(mockBlobOptions.Object); + + _blobStorageConverter = new BlobStorageConverter(workerOptions, mockBlobOptionsSnapshot.Object, logger); + } + + [Fact] + public async Task ConvertAsync_BlobContainerClient_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobContainerClient), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Name).Returns("MyContainer"); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobContainerClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("MyContainer", clientResult.Name); + } + + [Fact] + public async Task ConvertAsync_BlobContainerClient_WithSubdirectoryPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "test"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobContainerClient), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Name).Returns("MyContainer"); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobContainerClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("MyContainer", clientResult.Name); + } + + [Fact] + public async Task ConvertAsync_BlobContainerClient_WithFilePath_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobContainerClient), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Name).Returns("MyContainer"); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Equal("Binding to a BlobContainerClient with a blob path is not supported. Either bind to the container path, or use BlobClient instead.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_BlobContainerClientCollection_List_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Name).Returns("MyContainer"); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Equal("Binding to a BlobContainerClient collection is not supported.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_BlobContainerClientCollection_Array_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobContainerClient[]), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Name).Returns("MyContainer"); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Equal("Binding to a BlobContainerClient collection is not supported.", conversionResult.Error.Message); + } + } +} diff --git a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterCoreTests.cs b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterCoreTests.cs new file mode 100644 index 000000000..952cb3cf7 --- /dev/null +++ b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterCoreTests.cs @@ -0,0 +1,191 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Blob +{ + public class BlobStorageConverterCoreTests + { + private readonly BlobStorageConverter _blobStorageConverter; + private readonly Mock _mockBlobServiceClient; + + public BlobStorageConverterCoreTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + + var workerOptions = host.Services.GetService>(); + var logger = host.Services.GetService>(); + + _mockBlobServiceClient = new Mock(); + + var mockBlobOptions = new Mock(); + mockBlobOptions.Object.Client = _mockBlobServiceClient.Object; + + var mockBlobOptionsSnapshot = new Mock>(); + mockBlobOptionsSnapshot + .Setup(m => m.Get(It.IsAny())) + .Returns(mockBlobOptions.Object); + + _blobStorageConverter = new BlobStorageConverter(workerOptions, mockBlobOptionsSnapshot.Object, logger); + } + + [Fact] + public async Task ConvertAsync_ContentSource_AsObject_ReturnsUnhandled() + { + // Arrange + var context = new TestConverterContext(typeof(BlobClient), new object()); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingData_Null_ReturnsUnhandled() + { + // Arrange + var context = new TestConverterContext(typeof(BlobClient), null); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ResultIsEmpty_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns((BlobClient)null); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unable to convert blob binding data to type 'BlobClient'.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ThrowsException_ReturnsFailure() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + _mockBlobServiceClient + .Setup(m => m.GetBlobContainerClient(It.IsAny())) + .Throws(new Exception()); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingDataSource_NotBlobExtension_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "anotherExtensions"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingDataContentType_Unsupported_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs", contentType: "binary"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unexpected content-type. Only 'application/json' is supported.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ModelBindingData_InvalidContent_Throws_ReturnsFailed() + { + // Arrange + string badJsonData = $@"{{ + ""Connection"" : ""Connection"", + ""ContainerName"" ""ContainerName"", + ""BlobName"" : ""BlobName"", + }}"; + + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(new BinaryData(badJsonData), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_MissingConnectionParameter_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(null), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("'Connection' cannot be null or empty.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_MissingContainerNameParameter_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(container: null), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var clientResult = (BlobClient)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("'ContainerName' cannot be null or empty.", conversionResult.Error.Message); + } + } +} diff --git a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs deleted file mode 100644 index 5474961df..000000000 --- a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterTests.cs +++ /dev/null @@ -1,825 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Azure; -using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Models; -using Google.Protobuf; -using Microsoft.Azure.Functions.Worker; -using Microsoft.Azure.Functions.Worker.Converters; -using Microsoft.Azure.Functions.Worker.Grpc.Messages; -using Microsoft.Azure.Functions.Worker.Tests.Converters; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Moq; -using Xunit; - -// Scenarios for BlobBaseClient, BlockBlobClient, PageBlobClient, and AppendBlobClient -// are tested via E2E tests as Moq does not support mocking extension methods directly -namespace Microsoft.Azure.Functions.WorkerExtension.Tests -{ - public class BlobStorageConverterTests - { - private readonly BlobStorageConverter _blobStorageConverter; - private readonly Mock _mockBlobServiceClient; - - public BlobStorageConverterTests() - { - var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); - - var workerOptions = host.Services.GetService>(); - var logger = host.Services.GetService>(); - - _mockBlobServiceClient = new Mock(); - - var mockBlobOptions = new Mock(); - mockBlobOptions.Object.Client = _mockBlobServiceClient.Object; - - var mockBlobOptionsSnapshot = new Mock>(); - mockBlobOptionsSnapshot - .Setup(m => m.Get(It.IsAny())) - .Returns(mockBlobOptions.Object); - - _blobStorageConverter = new BlobStorageConverter(workerOptions, mockBlobOptionsSnapshot.Object, logger); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_BlobClient_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - - var mockBlobClient = new Mock(); - mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var clientResult = (BlobClient)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.Equal("MyBlob", clientResult.Name); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_BlobClientCollection_List_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); - - var mockBlobClient = new Mock(); - mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var clientResult = (IEnumerable)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsAssignableFrom>(clientResult); - Assert.Equal("MyBlob", clientResult.First().Name); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_BlobClientCollection_Array_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(BlobClient[]), grpcModelBindingData); - - var mockBlobClient = new Mock(); - mockBlobClient.Setup(m => m.Name).Returns("MyBlob"); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var clientResult = (BlobClient[])conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(clientResult); - Assert.Equal("MyBlob", clientResult.First().Name); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_BlobContainerClient_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(BlobContainerClient), grpcModelBindingData); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.Name).Returns("MyContainer"); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var clientResult = (BlobContainerClient)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.Equal("MyContainer", clientResult.Name); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_BlobContainerClientCollection_List_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.Name).Returns("MyContainer"); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var clientResult = (IEnumerable)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsAssignableFrom>(clientResult); - Assert.Equal("MyContainer", clientResult.First().Name); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_BlobContainerClientCollection_Array_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(BlobContainerClient[]), grpcModelBindingData); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.Name).Returns("MyContainer"); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var clientResult = (BlobContainerClient[])conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(clientResult); - Assert.Equal("MyContainer", clientResult.First().Name); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_String_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(string), grpcModelBindingData); - - var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var stringResult = (string)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.Equal("MyBlobString", stringResult); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_StringCollection_List_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); - - var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var stringResult = (IEnumerable)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsAssignableFrom>(stringResult); - Assert.Equal("MyBlobString", stringResult.First().ToString()); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_StringCollection_Array_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(string[]), grpcModelBindingData); - - var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var stringResult = (string[])conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(stringResult); - Assert.Equal("MyBlobString", stringResult.First().ToString()); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_Stream_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(Stream), grpcModelBindingData); - - var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); - var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) - .ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var streamResult = (Stream)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.Equal(expectedStream, streamResult); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_StreamCollection_List_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); - - var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); - var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); - - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) - .ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var streamResult = (IEnumerable)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsAssignableFrom>(streamResult); - Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_StreamCollection_Array_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(Stream[]), grpcModelBindingData); - - var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); - var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); - - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) - .ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var streamResult = (Stream[])conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_ByteArray_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(byte[]), grpcModelBindingData); - - var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); - var testStream = new MemoryStream(expectedByteArray) - { - Position = 0 - }; - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadToAsync(It.IsAny(), default)) - .Callback(async (s, _) => await testStream.CopyToAsync(s)) - .ReturnsAsync(new Mock().Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var byteResult = (byte[])conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(byteResult); - Assert.Equal(new byte[0], byteResult); - // Assert.Equal(expectedByteArray, byteResult); // Running into issues mocking DownloadToAsync stream update - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_ByteArrayCollection_List_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); - - var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); - var testStream = new MemoryStream(expectedByteArray); - testStream.Position = 0; - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadToAsync(It.IsAny(), default)) - .Callback(async (s, _) => await testStream.CopyToAsync(s)) - .ReturnsAsync(new Mock().Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var byteResult = (IEnumerable)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsAssignableFrom>(byteResult); - // Assert.Equal(expectedByteArray, byteResult.First()); // Running into issues mocking DownloadToAsync stream update - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_ByteArrayCollection_Array_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(byte[][]), grpcModelBindingData); - - var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); - var testStream = new MemoryStream(expectedByteArray); - testStream.Position = 0; - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadToAsync(It.IsAny(), default)) - .Callback(async (s, _) => await testStream.CopyToAsync(s)) - .ReturnsAsync(new Mock().Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var byteResult = (byte[][])conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(byteResult); - // Assert.Equal(expectedByteArray, byteResult.First()); // Running into issues mocking DownloadToAsync steam update - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_POCO_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(Book), grpcModelBindingData); - - var expectedBook = new Book() { Name = "MyBook" }; - - var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); - var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) - .ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var pocoResult = (Book)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.Equal(expectedBook.GetType(), pocoResult.GetType()); - Assert.Equal(expectedBook.Name, pocoResult.Name); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_POCOCollection_List_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); - - var expectedBookList = new List() { new Book() { Name = "MyBook" } }; - var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); - var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); - - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) - .ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var pocoResult = (IEnumerable)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsAssignableFrom>(pocoResult); - Assert.Equal(expectedBookList[0].Name, pocoResult.First().Name); - } - - [Fact] - public async Task ConvertAsync_ValidModelBindingData_POCOCollection_Array_ReturnsSuccess() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(Book[]), grpcModelBindingData); - - var expectedBookList = new Book[] { new Book() { Name = "MyBook" } }; - var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); - var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); - - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) - .ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var pocoResult = (Book[])conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.IsType(pocoResult); - Assert.Equal(expectedBookList[0].Name, pocoResult.First().Name); - } - - // Unhappy cases - - [Fact] - public async Task ConvertAsync_ContentSource_AsObject_ReturnsUnhandled() - { - // Arrange - var context = new TestConverterContext(typeof(BlobClient), new object()); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - - // Assert - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); - } - - [Fact] - public async Task ConvertAsync_ModelBindingData_Null_ReturnsUnhandled() - { - // Arrange - var context = new TestConverterContext(typeof(BlobClient), null); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - - // Assert - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); - } - - [Fact] - public async Task ConvertAsync_ThrowsException_ReturnsFailure() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - - _mockBlobServiceClient - .Setup(m => m.GetBlobContainerClient(It.IsAny())) - .Throws(new Exception()); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - - // Assert - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - } - - [Fact] - public async Task ConvertAsync_ModelBindingDataSource_NotBlobExtension_ReturnsFailed() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "anotherExtensions"); - var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - - // Assert - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - } - - [Fact] - public async Task ConvertAsync_ModelBindingDataContentType_Unsupported_ReturnsFailed() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs", contentType: "binary"); - var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - - // Assert - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.Equal("Unexpected content-type. Only 'application/json' is supported.", conversionResult.Error.Message); - } - - [Fact] - public async Task ConvertAsync_ModelBindingData_InvalidContent_Throws_ReturnsFailed() - { - // Arrange - string badJsonData = $@"{{ - ""Connection"" : ""Connection"", - ""ContainerName"" ""ContainerName"", - ""BlobName"" : ""BlobName"", - }}"; - - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(new BinaryData(badJsonData), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - - // Assert - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.IsType(conversionResult.Error); - } - - [Fact] - public async Task ConvertAsync_MissingConnectionParameter_ReturnsFailed() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(null), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var clientResult = (BlobClient)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.Equal("Value cannot be null. (Parameter 'connectionName')", conversionResult.Error.Message); - } - - [Fact] - public async Task ConvertAsync_MissingContainerNameParameter_ReturnsFailed() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(container: null), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var clientResult = (BlobClient)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.Equal("Value cannot be null. (Parameter 'containerName')", conversionResult.Error.Message); - } - - [Fact] - public async Task ConvertAsync_BlobClient_MissingBlobNameParameter_ReturnsFailed() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(blobName: null), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(BlobClient), grpcModelBindingData); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var clientResult = (BlobClient)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.Equal("Value cannot be null. (Parameter 'blobName')", conversionResult.Error.Message); - } - - [Fact] - public async Task ConvertAsync_POCO_InvalidJson_ReturnsFailed() - { - // Arrange - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "AzureStorageBlobs"); - var context = new TestConverterContext(typeof(Book), grpcModelBindingData); - - var expectedBook = new Book() { Name = "MyBook" }; - - var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name:\"MyBook\"}")); - var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) - .ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var pocoResult = (Book)conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.IsType(conversionResult.Error); - } - - [Fact] - public async Task ConvertAsync_IncorrectJsonContent_POCOCollection_Array_ReturnsFailed() - { - // Arrange - var grpcModelBindingData = GetTestGrpcCollectionModelBindingData(GetTestBinaryData()); - var context = new TestConverterContext(typeof(Book[]), grpcModelBindingData); - - var expectedBookList = new Book[] { new Book() { Name = "MyBook" } }; - var testStream = new MemoryStream(Encoding.UTF8.GetBytes("i should fail")); - var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); - - var mockResponse = new Mock>(); - mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); - - var mockBlobClient = new Mock(); - mockBlobClient - .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) - .ReturnsAsync(mockResponse.Object); - - var mockContainer = new Mock(); - mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); - - _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); - - // Act - var conversionResult = await _blobStorageConverter.ConvertAsync(context); - var pocoResult = (Book[])conversionResult.Value; - - // Assert - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.IsType(conversionResult.Error); - Assert.IsType(conversionResult.Error.InnerException); - } - - private BinaryData GetTestBinaryData(string connection = "Connection", string container = "Container", string blobName = "MyBlob") - { - string jsonData = $@"{{ - ""Connection"" : ""{connection}"", - ""ContainerName"" : ""{container}"", - ""BlobName"" : ""{blobName}"" - }}"; - - return new BinaryData(jsonData); - } - - private GrpcCollectionModelBindingData GetTestGrpcCollectionModelBindingData(BinaryData data) - { - var modelBindingData = new ModelBindingData() - { - Version = "1.0", - Source = "AzureStorageBlobs", - Content = ByteString.CopyFrom(data), - ContentType = "application/json" - }; - - var array = new CollectionModelBindingData(); - array.ModelBindingData.Add(modelBindingData); - - return new GrpcCollectionModelBindingData(array); - } - - public class Book - { - public string Name { get; set; } - } - - public interface TMock - { - } - } -} diff --git a/test/Worker.Extensions.Tests/Blob/BlobTestHelper.cs b/test/Worker.Extensions.Tests/Blob/BlobTestHelper.cs new file mode 100644 index 000000000..52c96b721 --- /dev/null +++ b/test/Worker.Extensions.Tests/Blob/BlobTestHelper.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Blob +{ + public class BlobTestHelper + { + public static BinaryData GetTestBinaryData(string connection = "Connection", string container = "Container", string blobName = "") + { + string jsonData = $@"{{ + ""Connection"" : ""{connection}"", + ""ContainerName"" : ""{container}"", + ""BlobName"" : ""{blobName}"" + }}"; + + return new BinaryData(jsonData); + } + } +} \ No newline at end of file diff --git a/test/Worker.Extensions.Tests/Blob/ByteArrayTests.cs b/test/Worker.Extensions.Tests/Blob/ByteArrayTests.cs new file mode 100644 index 000000000..ce26ade05 --- /dev/null +++ b/test/Worker.Extensions.Tests/Blob/ByteArrayTests.cs @@ -0,0 +1,319 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Blob +{ + public class ByteArrayTests + { + private readonly BlobStorageConverter _blobStorageConverter; + private readonly Mock _mockBlobServiceClient; + + public ByteArrayTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + + var workerOptions = host.Services.GetService>(); + var logger = host.Services.GetService>(); + + _mockBlobServiceClient = new Mock(); + + var mockBlobOptions = new Mock(); + mockBlobOptions.Object.Client = _mockBlobServiceClient.Object; + + var mockBlobOptionsSnapshot = new Mock>(); + mockBlobOptionsSnapshot + .Setup(m => m.Get(It.IsAny())) + .Returns(mockBlobOptions.Object); + + _blobStorageConverter = new BlobStorageConverter(workerOptions, mockBlobOptionsSnapshot.Object, logger); + } + + [Fact] + public async Task ConvertAsync_ByteArray_WithFilePath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(byte[]), grpcModelBindingData); + + var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); + var testStream = new MemoryStream(expectedByteArray) + { + Position = 0 + }; + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadToAsync(It.IsAny(), default)) + .Callback(async (s, _) => await testStream.CopyToAsync(s)) + .ReturnsAsync(new Mock().Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var byteResult = (byte[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(byteResult); + Assert.Equal(new byte[0], byteResult); + } + + [Fact] + public async Task ConvertAsync_ByteArray_FilePathWithoutFileExtension_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(byte[]), grpcModelBindingData); + + var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); + var testStream = new MemoryStream(expectedByteArray) + { + Position = 0 + }; + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadToAsync(It.IsAny(), default)) + .Callback(async (s, _) => await testStream.CopyToAsync(s)) + .ReturnsAsync(new Mock().Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var byteResult = (byte[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(byteResult); + Assert.Equal(new byte[0], byteResult); + } + + [Fact] + public async Task ConvertAsync_ByteArray_WithContainerPath_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(byte[]), grpcModelBindingData); + + var mockContainer = new Mock(); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Equal("'BlobName' cannot be null or empty when binding to a single blob.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ByteArrayCollection_List_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); + var testStream = new MemoryStream(expectedByteArray); + testStream.Position = 0; + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadToAsync(It.IsAny(), default)) + .Callback(async (s, _) => await testStream.CopyToAsync(s)) + .ReturnsAsync(new Mock().Object); + + var mockResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var byteResult = (IEnumerable)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(byteResult); + } + + [Fact] + public async Task ConvertAsync_ByteArrayCollection_Array_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(byte[][]), grpcModelBindingData); + + var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); + var testStream = new MemoryStream(expectedByteArray); + testStream.Position = 0; + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadToAsync(It.IsAny(), default)) + .Callback(async (s, _) => await testStream.CopyToAsync(s)) + .ReturnsAsync(new Mock().Object); + + var mockResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var byteResult = (byte[][])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(byteResult); + } + + [Fact] + public async Task ConvertAsync_ByteArrayCollection_WithSubdirectoryPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "test"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(byte[][]), grpcModelBindingData); + + var expectedByteArray = Encoding.UTF8.GetBytes("MyBlobByteArray"); + var testStream = new MemoryStream(expectedByteArray); + testStream.Position = 0; + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadToAsync(It.IsAny(), default)) + .Callback(async (s, _) => await testStream.CopyToAsync(s)) + .ReturnsAsync(new Mock().Object); + + var mockResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var byteResult = (byte[][])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(byteResult); + } + + [Fact] + public async Task ConvertAsync_ByteArrayCollection_WithFilePath_ValidContent_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(byte[][]), grpcModelBindingData); + + var jsonString = JsonConvert.SerializeObject(new List { Encoding.UTF8.GetBytes("Item1"), Encoding.UTF8.GetBytes("Item2") }); + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var byteResult = (byte[][])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(byteResult); + Assert.Equal("Item1", Encoding.UTF8.GetString(byteResult[0])); + } + + [Fact] + public async Task ConvertAsync_ByteArrayCollection_WithFilePath_InvalidContent_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(byte[][]), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("[1,2]")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); + } + } +} diff --git a/test/Worker.Extensions.Tests/Blob/POCOTests.cs b/test/Worker.Extensions.Tests/Blob/POCOTests.cs new file mode 100644 index 000000000..202d7b221 --- /dev/null +++ b/test/Worker.Extensions.Tests/Blob/POCOTests.cs @@ -0,0 +1,368 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Blob +{ + public class POCOTests + { + private readonly BlobStorageConverter _blobStorageConverter; + private readonly Mock _mockBlobServiceClient; + + public POCOTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + + var workerOptions = host.Services.GetService>(); + var logger = host.Services.GetService>(); + + _mockBlobServiceClient = new Mock(); + + var mockBlobOptions = new Mock(); + mockBlobOptions.Object.Client = _mockBlobServiceClient.Object; + + var mockBlobOptionsSnapshot = new Mock>(); + mockBlobOptionsSnapshot + .Setup(m => m.Get(It.IsAny())) + .Returns(mockBlobOptions.Object); + + _blobStorageConverter = new BlobStorageConverter(workerOptions, mockBlobOptionsSnapshot.Object, logger); + } + + [Fact] + public async Task ConvertAsync_POCO_WithFilePath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book), grpcModelBindingData); + + var expectedBook = new Book() { Name = "MyBook" }; + + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (Book)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedBook.GetType(), pocoResult.GetType()); + Assert.Equal(expectedBook.Name, pocoResult.Name); + } + + [Fact] + public async Task ConvertAsync_POCO_FilePathWithoutFileExtension_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book), grpcModelBindingData); + + var expectedBook = new Book() { Name = "MyBook" }; + + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (Book)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedBook.GetType(), pocoResult.GetType()); + Assert.Equal(expectedBook.Name, pocoResult.Name); + } + + [Fact] + public async Task ConvertAsync_POCO_WithContainerPath_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book), grpcModelBindingData); + + var mockContainer = new Mock(); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Equal("'BlobName' cannot be null or empty when binding to a single blob.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_POCO_InvalidJson_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book), grpcModelBindingData); + + var expectedBook = new Book() { Name = "MyBook" }; + + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name:\"MyBook\"}")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (Book)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_POCOCollection_List_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var expectedBookList = new List() { new Book() { Name = "MyBook" } }; + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List { BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (IEnumerable)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(pocoResult); + Assert.Equal(expectedBookList[0].Name, pocoResult.First().Name); + } + + [Fact] + public async Task ConvertAsync_POCOCollection_Array_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book[]), grpcModelBindingData); + + var expectedBookList = new Book[] { new Book() { Name = "MyBook" } }; + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("{\"Name\":\"MyBook\"}")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List { BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var pocoResult = (Book[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(pocoResult); + Assert.Equal(expectedBookList[0].Name, pocoResult.First().Name); + } + + [Fact] + public async Task ConvertAsync_POCOCollection_InvalidJson_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book[]), grpcModelBindingData); + + var expectedBookList = new Book[] { new Book() { Name = "MyBook" } }; + var testStream = new MemoryStream(Encoding.UTF8.GetBytes("i should fail")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(testStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var blobItemMockResponse = new Mock(); + var expectedOutput = Page.FromValues(new List { BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, blobItemMockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_POCOCollection_WithFilePath_ValidContent_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book[]), grpcModelBindingData); + + var jsonString = JsonConvert.SerializeObject(new List { new { Name = "MyBook" }, new { Name = "MySecondBook" }}); + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var bookResult = (Book[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(bookResult); + Assert.Equal("MyBook", bookResult[0].Name); + Assert.Equal("MySecondBook", bookResult[1].Name); + } + + [Fact] + public async Task ConvertAsync_POCOCollection_WithFilePath_InvalidContent_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Book[]), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("[1,2]")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); + } + + public class Book + { + public string Name { get; set; } + } + } +} diff --git a/test/Worker.Extensions.Tests/Blob/StreamTests.cs b/test/Worker.Extensions.Tests/Blob/StreamTests.cs new file mode 100644 index 000000000..356e5fcde --- /dev/null +++ b/test/Worker.Extensions.Tests/Blob/StreamTests.cs @@ -0,0 +1,279 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Blob +{ + public class StreamTests + { + private readonly BlobStorageConverter _blobStorageConverter; + private readonly Mock _mockBlobServiceClient; + + public StreamTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + + var workerOptions = host.Services.GetService>(); + var logger = host.Services.GetService>(); + + _mockBlobServiceClient = new Mock(); + + var mockBlobOptions = new Mock(); + mockBlobOptions.Object.Client = _mockBlobServiceClient.Object; + + var mockBlobOptionsSnapshot = new Mock>(); + mockBlobOptionsSnapshot + .Setup(m => m.Get(It.IsAny())) + .Returns(mockBlobOptions.Object); + + _blobStorageConverter = new BlobStorageConverter(workerOptions, mockBlobOptionsSnapshot.Object, logger); + } + + [Fact] + public async Task ConvertAsync_Stream_WithFilePath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Stream), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var streamResult = (Stream)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedStream, streamResult); + } + + [Fact] + public async Task ConvertAsync_Stream_FilePathWithoutFileExtension_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Stream), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var streamResult = (Stream)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedStream, streamResult); + } + + [Fact] + public async Task ConvertAsync_Stream_WithContainerPath_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Stream), grpcModelBindingData); + + var mockContainer = new Mock(); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Equal("'BlobName' cannot be null or empty when binding to a single blob.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_StreamCollection_List_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var streamResult = (IEnumerable)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(streamResult); + Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); + } + + [Fact] + public async Task ConvertAsync_StreamCollection_Array_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Stream[]), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var streamResult = (Stream[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); + } + + [Fact] + public async Task ConvertAsync_StreamCollection_WithSubdirectoryPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "test"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Stream[]), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("MyBlobStream")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var streamResult = (Stream[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedStream.ToString(), streamResult.First().ToString()); + } + + [Fact] + public async Task ConvertAsync_StreamCollection_WithFilePath_Throws_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "test.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(Stream[]), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("[{\"name\": \"Stream1\"}]")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List{ BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Contains("Deserialization of types without a parameterless constructor, a singular parameterized constructor, or a parameterized constructor annotated with 'JsonConstructorAttribute' is not supported.", conversionResult.Error.Message); + } + } +} diff --git a/test/Worker.Extensions.Tests/Blob/StringTests.cs b/test/Worker.Extensions.Tests/Blob/StringTests.cs new file mode 100644 index 000000000..5df157199 --- /dev/null +++ b/test/Worker.Extensions.Tests/Blob/StringTests.cs @@ -0,0 +1,268 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Blob +{ + public class StringTests + { + private readonly BlobStorageConverter _blobStorageConverter; + private readonly Mock _mockBlobServiceClient; + + public StringTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + + var workerOptions = host.Services.GetService>(); + var logger = host.Services.GetService>(); + + _mockBlobServiceClient = new Mock(); + + var mockBlobOptions = new Mock(); + mockBlobOptions.Object.Client = _mockBlobServiceClient.Object; + + var mockBlobOptionsSnapshot = new Mock>(); + mockBlobOptionsSnapshot + .Setup(m => m.Get(It.IsAny())) + .Returns(mockBlobOptions.Object); + + _blobStorageConverter = new BlobStorageConverter(workerOptions, mockBlobOptionsSnapshot.Object, logger); + } + + [Fact] + public async Task ConvertAsync_String_WithFilePath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(string), grpcModelBindingData); + + var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var stringResult = (string)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("MyBlobString", stringResult); + } + + [Fact] + public async Task ConvertAsync_String_FilePathWithoutFileExtension_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(string), grpcModelBindingData); + + var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var stringResult = (string)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("MyBlobString", stringResult); + } + + [Fact] + public async Task ConvertAsync_String_WithContainerPath_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(string), grpcModelBindingData); + + var mockContainer = new Mock(); + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Equal("'BlobName' cannot be null or empty when binding to a single blob.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_StringCollection_List_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List { BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var stringResult = (IEnumerable)conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsAssignableFrom>(stringResult); + Assert.Equal("MyBlobString", stringResult.First().ToString()); + } + + [Fact] + public async Task ConvertAsync_StringCollection_Array_WithContainerPath_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(string[]), grpcModelBindingData); + + var blobDownloadResult = BlobsModelFactory.BlobDownloadResult(new BinaryData("MyBlobString")); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient.Setup(m => m.DownloadContentAsync()).ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List { BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var stringResult = (string[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(stringResult); + Assert.Equal("MyBlobString", stringResult.First().ToString()); + } + + [Fact] + public async Task ConvertAsync_StringCollection_WithFilePath_ValidContent_ReturnsSuccess() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(string[]), grpcModelBindingData); + + var jsonString = JsonConvert.SerializeObject(new List { "1", "2" }); + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List { BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + var stringResult = (string[])conversionResult.Value; + + // Assert + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.IsType(stringResult); + Assert.Equal("1", stringResult.First().ToString()); + Assert.Equal("2", stringResult.Last().ToString()); + } + + [Fact] + public async Task ConvertAsync_StringCollection_WithFilePath_InvalidContent_Throws_ReturnsFailed() + { + // Arrange + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(BlobTestHelper.GetTestBinaryData(blobName: "MyBlob.txt"), "AzureStorageBlobs"); + var context = new TestConverterContext(typeof(string[]), grpcModelBindingData); + + var expectedStream = new MemoryStream(Encoding.UTF8.GetBytes("[1,2]")); + var blobDownloadResult = BlobsModelFactory.BlobDownloadStreamingResult(expectedStream); + var mockResponse = new Mock>(); + mockResponse.SetupGet(r => r.Value).Returns(blobDownloadResult); + + var mockBlobClient = new Mock(); + mockBlobClient + .Setup(m => m.DownloadStreamingAsync(It.IsAny(), It.IsAny(), It.IsAny(), null, default)) + .ReturnsAsync(mockResponse.Object); + + var mockBlobItemResponse = new Mock(); + var expectedOutput = Page.FromValues(new List { BlobsModelFactory.BlobItem("MyBlob") }, continuationToken: null, mockBlobItemResponse.Object); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.GetBlobClient(It.IsAny())).Returns(mockBlobClient.Object); + mockContainer.Setup(m => m.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(AsyncPageable.FromPages(new List> { expectedOutput })); + + _mockBlobServiceClient.Setup(m => m.GetBlobContainerClient(It.IsAny())).Returns(mockContainer.Object); + + // Act + var conversionResult = await _blobStorageConverter.ConvertAsync(context); + + // Assert + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.IsType(conversionResult.Error); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); + } + } +} diff --git a/test/Worker.Extensions.Tests/GrpcTestHelper.cs b/test/Worker.Extensions.Tests/GrpcTestHelper.cs new file mode 100644 index 000000000..634f1dc26 --- /dev/null +++ b/test/Worker.Extensions.Tests/GrpcTestHelper.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Google.Protobuf; +using Microsoft.Azure.Functions.Worker.Grpc.Messages; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests +{ + public static class GrpcTestHelper + { + internal static GrpcModelBindingData GetTestGrpcModelBindingData(BinaryData content, string source, string contentType = "application/json") + { + var data = new ModelBindingData() + { + Version = "1.0", + Source = source, + Content = ByteString.CopyFrom(content), + ContentType = contentType + }; + + return new GrpcModelBindingData(data); + } + } +} \ No newline at end of file From aa3ffd87b86410fbe7662466f07e10bdb19d0191 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 19 Jul 2023 10:17:10 -0700 Subject: [PATCH 24/47] Update shared extension exceptions to include actual value --- .../Exceptions/InvalidBindingSourceException.cs | 14 +++++++++----- .../Exceptions/InvalidContentTypeException.cs | 14 +++++++++----- .../src/BlobStorageConverter.cs | 4 ++-- .../Blob/BlobStorageConverterCoreTests.cs | 4 ++-- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/extensions/Worker.Extensions.Shared/Exceptions/InvalidBindingSourceException.cs b/extensions/Worker.Extensions.Shared/Exceptions/InvalidBindingSourceException.cs index f0bbee0b6..b5ea0baa4 100644 --- a/extensions/Worker.Extensions.Shared/Exceptions/InvalidBindingSourceException.cs +++ b/extensions/Worker.Extensions.Shared/Exceptions/InvalidBindingSourceException.cs @@ -13,16 +13,20 @@ internal class InvalidBindingSourceException : InvalidOperationException /// /// Initializes a new instance of the InvalidBindingSourceException class with a specified error message. /// - /// The source(s) that is supported. - public InvalidBindingSourceException(string source) : base($"Unexpected binding source. Only '{source}' is supported.") { } + /// The source that is being provided by ModelBindingData + /// The source(s) that is supported. + public InvalidBindingSourceException(string actualSource, string expectedSource) + : base($"Unexpected binding source '{actualSource}'. Only '{expectedSource}' is supported.") { } /// /// Initializes a new instance of the InvalidBindingSourceException class with a specified error message /// and a reference to the inner exception that is the cause of this exception. /// - /// The source(s) that is supported. + /// The source that is being provided by ModelBindingData + /// The source(s) that is supported. /// The exception that is the cause of the current exception - /// or a null reference if no inner exception is specified.. - public InvalidBindingSourceException(string source, Exception? innerException) : base($"Unexpected binding source. Only '{source}' is supported.", innerException) { } + /// or a null reference if no inner exception is specified. + public InvalidBindingSourceException(string actualSource, string expectedSource, Exception? innerException) + : base($"Unexpected binding source '{actualSource}'. Only '{expectedSource}' is supported.", innerException) { } } } diff --git a/extensions/Worker.Extensions.Shared/Exceptions/InvalidContentTypeException.cs b/extensions/Worker.Extensions.Shared/Exceptions/InvalidContentTypeException.cs index abfbb4999..417d5bfca 100644 --- a/extensions/Worker.Extensions.Shared/Exceptions/InvalidContentTypeException.cs +++ b/extensions/Worker.Extensions.Shared/Exceptions/InvalidContentTypeException.cs @@ -13,16 +13,20 @@ internal class InvalidContentTypeException : InvalidOperationException /// /// Initializes a new instance of the InvalidContentTypeException class with a specified error message. /// - /// The content type(s) that is supported. - public InvalidContentTypeException(string contentType) : base($"Unexpected content-type. Only '{contentType}' is supported.") { } + /// The source that is being provided. + /// The content type(s) that is supported. + public InvalidContentTypeException(string actualContentType, string expectedContentType) + : base($"Unexpected content-type '{actualContentType}'. Only '{expectedContentType}' is supported.") { } /// /// Initializes a new instance of the InvalidContentTypeException class with a specified error message /// and a reference to the inner exception that is the cause of this exception. /// - /// The content type(s) that is supported. + /// The source that is being provided. + /// The content type(s) that is supported. /// The exception that is the cause of the current exception - /// or a null reference if no inner exception is specified.. - public InvalidContentTypeException(string contentType, Exception innerException) : base($"Unexpected content-type. Only '{contentType}' is supported.", innerException) { } + /// or a null reference if no inner exception is specified. + public InvalidContentTypeException(string actualContentType, string expectedContentType, Exception innerException) + : base($"Unexpected content-type '{actualContentType}'. Only '{expectedContentType}' is supported.", innerException) { } } } diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs index a290d8a92..35f69f035 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/BlobStorageConverter.cs @@ -56,7 +56,7 @@ private async ValueTask ConvertFromBindingDataAsync(ConverterC { if (modelBindingData.Source is not Constants.BlobExtensionName) { - throw new InvalidBindingSourceException(Constants.BlobExtensionName); + throw new InvalidBindingSourceException(modelBindingData.Source, Constants.BlobExtensionName); } BlobBindingData blobData = GetBindingDataContent(modelBindingData); @@ -94,7 +94,7 @@ private BlobBindingData GetBindingDataContent(ModelBindingData bindingData) return bindingData.ContentType switch { Constants.JsonContentType => bindingData.Content.ToObjectFromJson(), - _ => throw new InvalidContentTypeException(Constants.JsonContentType) + _ => throw new InvalidContentTypeException(bindingData.ContentType, Constants.JsonContentType) }; } diff --git a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterCoreTests.cs b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterCoreTests.cs index 952cb3cf7..0139d902b 100644 --- a/test/Worker.Extensions.Tests/Blob/BlobStorageConverterCoreTests.cs +++ b/test/Worker.Extensions.Tests/Blob/BlobStorageConverterCoreTests.cs @@ -4,7 +4,6 @@ using System; using System.Threading.Tasks; using Azure.Storage.Blobs; -using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Tests.Converters; using Microsoft.Extensions.DependencyInjection; @@ -117,6 +116,7 @@ public async Task ConvertAsync_ModelBindingDataSource_NotBlobExtension_ReturnsFa // Assert Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unexpected binding source 'anotherExtensions'. Only 'AzureStorageBlobs' is supported.", conversionResult.Error.Message); } [Fact] @@ -131,7 +131,7 @@ public async Task ConvertAsync_ModelBindingDataContentType_Unsupported_ReturnsFa // Assert Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.Equal("Unexpected content-type. Only 'application/json' is supported.", conversionResult.Error.Message); + Assert.Equal("Unexpected content-type 'binary'. Only 'application/json' is supported.", conversionResult.Error.Message); } [Fact] From 1a563134584780f1203e891fbb58690a0fa48967 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Thu, 20 Jul 2023 09:30:18 -0700 Subject: [PATCH 25/47] Fix BlobStorageBindingOptions client cache (#1764) --- .../src/Config/BlobStorageBindingOptions.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptions.cs b/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptions.cs index 92662bbd4..edcd79fa2 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptions.cs +++ b/extensions/Worker.Extensions.Storage.Blobs/src/Config/BlobStorageBindingOptions.cs @@ -21,17 +21,21 @@ internal class BlobStorageBindingOptions internal BlobServiceClient CreateClient() { - if (this.Client is not null) + if (Client is not null) { - return this.Client; + return Client; } if (ServiceUri is not null && Credential is not null) { - return new BlobServiceClient(ServiceUri, Credential, BlobClientOptions); + Client = new BlobServiceClient(ServiceUri, Credential, BlobClientOptions); + } + else + { + Client = new BlobServiceClient(ConnectionString, BlobClientOptions); } - return new BlobServiceClient(ConnectionString, BlobClientOptions); + return Client; } } } \ No newline at end of file From d12561dcf738a221c907a9d4a511a0b60d40090e Mon Sep 17 00:00:00 2001 From: sarah <35204912+satvu@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:49:03 -0700 Subject: [PATCH 26/47] Create ASP.NET integration sample app (#1682) --- DotNetWorker.sln | 7 ++++ .../AspNetIntegration.csproj | 36 +++++++++++++++++++ samples/AspNetIntegration/Program.cs | 32 +++++++++++++++++ .../AspNetIntegration/RoutingMiddleware.cs | 34 ++++++++++++++++++ .../SimpleHttpTrigger/SimpleHttpTrigger.cs | 20 +++++++++++ samples/AspNetIntegration/host.json | 11 ++++++ samples/AspNetIntegration/local.settings.json | 8 +++++ 7 files changed, 148 insertions(+) create mode 100644 samples/AspNetIntegration/AspNetIntegration.csproj create mode 100644 samples/AspNetIntegration/Program.cs create mode 100644 samples/AspNetIntegration/RoutingMiddleware.cs create mode 100644 samples/AspNetIntegration/SimpleHttpTrigger/SimpleHttpTrigger.cs create mode 100644 samples/AspNetIntegration/host.json create mode 100644 samples/AspNetIntegration/local.settings.json diff --git a/DotNetWorker.sln b/DotNetWorker.sln index dac4af694..a0ce8d085 100644 --- a/DotNetWorker.sln +++ b/DotNetWorker.sln @@ -132,6 +132,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.SignalRSe EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.Tests", "test\Worker.Extensions.Tests\Worker.Extensions.Tests.csproj", "{17BDCE12-6964-4B87-B2AC-68CE270A3E9A}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AspNetIntegration", "samples\AspNetIntegration\AspNetIntegration.csproj", "{D2F67410-9933-42E8-B04A-E17634D83A30}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -326,6 +328,10 @@ Global {17BDCE12-6964-4B87-B2AC-68CE270A3E9A}.Debug|Any CPU.Build.0 = Debug|Any CPU {17BDCE12-6964-4B87-B2AC-68CE270A3E9A}.Release|Any CPU.ActiveCfg = Release|Any CPU {17BDCE12-6964-4B87-B2AC-68CE270A3E9A}.Release|Any CPU.Build.0 = Release|Any CPU + {D2F67410-9933-42E8-B04A-E17634D83A30}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D2F67410-9933-42E8-B04A-E17634D83A30}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D2F67410-9933-42E8-B04A-E17634D83A30}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D2F67410-9933-42E8-B04A-E17634D83A30}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -383,6 +389,7 @@ Global {1F6B7CF6-0CC8-4C7F-825F-74B0BEC1CF0A} = {A7B4FF1E-3DF7-4F28-9333-D0961CDDF702} {286F9EE3-00AE-4EFA-BFD8-A2E58BC809D2} = {FD7243E4-BF18-43F8-8744-BA1D17ACF378} {17BDCE12-6964-4B87-B2AC-68CE270A3E9A} = {FD7243E4-BF18-43F8-8744-BA1D17ACF378} + {D2F67410-9933-42E8-B04A-E17634D83A30} = {9D6603BD-7EA2-4D11-A69C-0D9E01317FD6} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {497D2ED4-A13E-4BCA-8D29-F30CA7D0EA4A} diff --git a/samples/AspNetIntegration/AspNetIntegration.csproj b/samples/AspNetIntegration/AspNetIntegration.csproj new file mode 100644 index 000000000..5a5bfa51f --- /dev/null +++ b/samples/AspNetIntegration/AspNetIntegration.csproj @@ -0,0 +1,36 @@ + + + net7.0 + v4 + Exe + enable + enable + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + Never + + + + + + + + \ No newline at end of file diff --git a/samples/AspNetIntegration/Program.cs b/samples/AspNetIntegration/Program.cs new file mode 100644 index 000000000..c837ed100 --- /dev/null +++ b/samples/AspNetIntegration/Program.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license informations + +#define ENABLE_MIDDLEWARE + +using AspNetIntegration; +using Microsoft.Extensions.Hosting; + +#if ENABLE_MIDDLEWARE + var host = new HostBuilder() + .ConfigureFunctionsWebApplication(builder => + { + // can still register middleware and use this extension method the same way + // .ConfigureFunctionsWorkerDefaults() is used + builder.UseWhen((context)=> + { + // We want to use this middleware only for http trigger invocations. + return context.FunctionDefinition.InputBindings.Values + .First(a => a.Type.EndsWith("Trigger")).Type == "httpTrigger"; + }); + }) + .Build(); + host.Run(); +#else + // + var host = new HostBuilder() + .ConfigureFunctionsWebApplication() + .Build(); + + host.Run(); + // +#endif diff --git a/samples/AspNetIntegration/RoutingMiddleware.cs b/samples/AspNetIntegration/RoutingMiddleware.cs new file mode 100644 index 000000000..851c638e7 --- /dev/null +++ b/samples/AspNetIntegration/RoutingMiddleware.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Middleware; + +namespace AspNetIntegration +{ + internal class RoutingMiddleware : IFunctionsWorkerMiddleware + { + public Task Invoke(FunctionContext context, FunctionExecutionDelegate next) + { + // retrieve http context from function context + HttpContext httpContext = context.GetHttpContext() + ?? throw new InvalidOperationException($"{nameof(context)} has no http context associated with it."); + + // operations can be performed using HttpContext + // example of getting route information from HttpContext: + RouteEndpoint? endpoint = httpContext.GetEndpoint() as RouteEndpoint; + + if (endpoint != null) + { + string? displayName = endpoint.DisplayName; + string? routePattern = endpoint.RoutePattern.RawText; + IReadOnlyList? httpMethods = (endpoint.Metadata.Single() as HttpMethodMetadata)?.HttpMethods; + } + + // continue along with function execution + return next(context); + } + } +} diff --git a/samples/AspNetIntegration/SimpleHttpTrigger/SimpleHttpTrigger.cs b/samples/AspNetIntegration/SimpleHttpTrigger/SimpleHttpTrigger.cs new file mode 100644 index 000000000..b7f25ee5a --- /dev/null +++ b/samples/AspNetIntegration/SimpleHttpTrigger/SimpleHttpTrigger.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.Functions.Worker; + +namespace AspNetIntegration +{ + public class SimpleHttpTrigger + { + // + [Function("SimpleHttpTrigger")] + public IActionResult Run([HttpTrigger(AuthorizationLevel.Anonymous, "get")] HttpRequest req) + { + return new OkObjectResult("Welcome to Azure Functions!"); + } + // + } +} diff --git a/samples/AspNetIntegration/host.json b/samples/AspNetIntegration/host.json new file mode 100644 index 000000000..beb2e4020 --- /dev/null +++ b/samples/AspNetIntegration/host.json @@ -0,0 +1,11 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + } +} \ No newline at end of file diff --git a/samples/AspNetIntegration/local.settings.json b/samples/AspNetIntegration/local.settings.json new file mode 100644 index 000000000..fa72d2ea4 --- /dev/null +++ b/samples/AspNetIntegration/local.settings.json @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", + "AzureWebJobsFeatureFlags": "EnableHttpProxying" + } +} \ No newline at end of file From f10d01c4a296db15310b34695e11bac2ce6a3121 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari <37918412+aishwaryabh@users.noreply.github.com> Date: Fri, 16 Jun 2023 16:52:22 -0700 Subject: [PATCH 27/47] Implement SDK-Binding EventGrid Converter (#1589) --- .../src/EventGridTriggerAttribute.cs | 21 +++- .../src/Properties/AssemblyInfo.cs | 6 +- .../EventGridBinaryDataConverter.cs | 78 +++++++++++++ .../EventGridCloudEventConverter.cs | 33 ++++++ .../TypeConverters/EventGridEventConverter.cs | 33 ++++++ .../src/TypeConverters/EventGridHelper.cs | 42 +++++++ .../EventGridStringArrayConverter.cs | 64 ++++++++++ .../src/Worker.Extensions.EventGrid.csproj | 8 +- .../src/release_notes.md | 9 ++ .../EventGridTriggerBindingSamples.cs | 105 +++++++++++++++++ .../WorkerBindingSamples.csproj | 1 + test/SdkE2ETests/PublishTests.cs | 3 + .../EventGridBinaryDataConverterTests.cs | 96 +++++++++++++++ .../EventGridCloudEventConverterTests.cs | 102 ++++++++++++++++ .../EventGrid/EventGridEventConverterTests.cs | 110 ++++++++++++++++++ .../EventGridStringArrayConverterTests.cs | 65 +++++++++++ .../Worker.Extensions.Tests.csproj | 1 + 17 files changed, 773 insertions(+), 4 deletions(-) create mode 100644 extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridBinaryDataConverter.cs create mode 100644 extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridCloudEventConverter.cs create mode 100644 extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridEventConverter.cs create mode 100644 extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridHelper.cs create mode 100644 extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridStringArrayConverter.cs create mode 100644 extensions/Worker.Extensions.EventGrid/src/release_notes.md create mode 100644 samples/WorkerBindingSamples/EventGrid/EventGridTriggerBindingSamples.cs create mode 100644 test/Worker.Extensions.Tests/EventGrid/EventGridBinaryDataConverterTests.cs create mode 100644 test/Worker.Extensions.Tests/EventGrid/EventGridCloudEventConverterTests.cs create mode 100644 test/Worker.Extensions.Tests/EventGrid/EventGridEventConverterTests.cs create mode 100644 test/Worker.Extensions.Tests/EventGrid/EventGridStringArrayConverterTests.cs diff --git a/extensions/Worker.Extensions.EventGrid/src/EventGridTriggerAttribute.cs b/extensions/Worker.Extensions.EventGrid/src/EventGridTriggerAttribute.cs index 3aa4a6122..b0842fd0a 100644 --- a/extensions/Worker.Extensions.EventGrid/src/EventGridTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventGrid/src/EventGridTriggerAttribute.cs @@ -1,11 +1,30 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters; namespace Microsoft.Azure.Functions.Worker { + [AllowConverterFallback(true)] + [InputConverter(typeof(EventGridCloudEventConverter))] + [InputConverter(typeof(EventGridEventConverter))] + [InputConverter(typeof(EventGridBinaryDataConverter))] + [InputConverter(typeof(EventGridStringArrayConverter))] public sealed class EventGridTriggerAttribute : TriggerBindingAttribute { + private bool _isBatched = false; + + /// + /// Gets or sets the configuration to enable batch processing of event grid. Default value is "false". + /// + [DefaultValue(false)] + public bool IsBatched + { + get => _isBatched; + set => _isBatched = value; + } + } } diff --git a/extensions/Worker.Extensions.EventGrid/src/Properties/AssemblyInfo.cs b/extensions/Worker.Extensions.EventGrid/src/Properties/AssemblyInfo.cs index ea38d15e7..974875917 100644 --- a/extensions/Worker.Extensions.EventGrid/src/Properties/AssemblyInfo.cs +++ b/extensions/Worker.Extensions.EventGrid/src/Properties/AssemblyInfo.cs @@ -1,6 +1,8 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System.Runtime.CompilerServices; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; -[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.EventGrid", "3.2.1")] +[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.EventGrid", "3.3.0")] +[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridBinaryDataConverter.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridBinaryDataConverter.cs new file mode 100644 index 000000000..d55d9e28d --- /dev/null +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridBinaryDataConverter.cs @@ -0,0 +1,78 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Converters; + +namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters +{ + /// + /// Converter to bind to BinaryData or BinaryData[] parameter. + /// + [SupportedConverterType(typeof(BinaryData))] + [SupportedConverterType(typeof(BinaryData[]))] + internal class EventGridBinaryDataConverter : IInputConverter + { + public ValueTask ConvertAsync(ConverterContext context) + { + try + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + if (context.Source is not string contextSource) + { + return new(ConversionResult.Failed(new InvalidOperationException("Context source must be a non-null string. Current type of context source is " + context?.Source?.GetType()))); + } + + var targetType = context.TargetType; + + switch (targetType) + { + case Type t when t == typeof(BinaryData): + return new(ConversionResult.Success((BinaryData.FromString(contextSource)))); + case Type t when t == typeof(BinaryData[]): + return new(ConversionResult.Success(ConvertToBinaryDataArray(contextSource))); + } + } + catch (JsonException ex) + { + string msg = String.Format(CultureInfo.CurrentCulture, + @"Binding parameters to complex objects uses JSON serialization. + 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or + 2. Change the event payload to be valid json."); + + return new(ConversionResult.Failed(new InvalidOperationException(msg, ex))); + } + catch (Exception ex) + { + return new(ConversionResult.Failed(ex)); + } + + return new(ConversionResult.Unhandled()); + } + + private BinaryData?[]? ConvertToBinaryDataArray(string contextSource) + { + var jsonData = JsonSerializer.Deserialize(contextSource, typeof(List)) as List; + List binaryDataList = new List(); + + if (jsonData is not null) + { + foreach (var item in jsonData) + { + var binaryData = item == null? null: BinaryData.FromString(item.ToString()); + binaryDataList.Add(binaryData); + } + } + + return binaryDataList.ToArray(); + } + } +} diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridCloudEventConverter.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridCloudEventConverter.cs new file mode 100644 index 000000000..67e4e3eab --- /dev/null +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridCloudEventConverter.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Azure.Messaging; +using Microsoft.Azure.Functions.Worker.Converters; + +namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters +{ + /// + /// Converter to bind to CloudEvent or CloudEvent[] parameter. + /// + [SupportedConverterType(typeof(CloudEvent))] + [SupportedConverterType(typeof(CloudEvent[]))] + internal class EventGridCloudEventConverter: IInputConverter + { + public ValueTask ConvertAsync(ConverterContext context) + { + if (context is null) + { + return new(ConversionResult.Failed(new ArgumentNullException(nameof(context)))); + } + + if (context.TargetType != typeof(CloudEvent) && context.TargetType != typeof(CloudEvent[])) + { + return new(ConversionResult.Unhandled()); + } + + return EventGridHelper.DeserializeToTargetType(context); + } + } +} diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridEventConverter.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridEventConverter.cs new file mode 100644 index 000000000..7397d2278 --- /dev/null +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridEventConverter.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Azure.Messaging.EventGrid; +using Microsoft.Azure.Functions.Worker.Converters; + +namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters +{ + /// + /// Converter to bind to EventGridEvent or EventGridEvent[] parameter. + /// + [SupportedConverterType(typeof(EventGridEvent))] + [SupportedConverterType(typeof(EventGridEvent[]))] + internal class EventGridEventConverter : IInputConverter + { + public ValueTask ConvertAsync(ConverterContext context) + { + if (context is null) + { + return new(ConversionResult.Failed(new ArgumentNullException(nameof(context)))); + } + + if (context?.TargetType != typeof(EventGridEvent) && context?.TargetType != typeof(EventGridEvent[])) + { + return new(ConversionResult.Unhandled()); + } + + return EventGridHelper.DeserializeToTargetType(context); + } + } +} diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridHelper.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridHelper.cs new file mode 100644 index 000000000..9347e21f3 --- /dev/null +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridHelper.cs @@ -0,0 +1,42 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Converters; + +namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters +{ + internal static class EventGridHelper + { + internal static ValueTask DeserializeToTargetType(ConverterContext context) + { + try + { + if (context.Source is not string contextSource) + { + return new(ConversionResult.Failed(new InvalidOperationException("Context source must be a non-null string. Current type of context source is " + context?.Source?.GetType()))); + } + + var targetType = context.TargetType; + var item = JsonSerializer.Deserialize(contextSource, targetType); + return new(ConversionResult.Success(item)); + } + catch (JsonException ex) + { + string msg = String.Format(CultureInfo.CurrentCulture, + @"Binding parameters to complex objects uses JSON serialization. + 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or + 2. Change the event payload to be valid json."); + + return new(ConversionResult.Failed(new InvalidOperationException(msg, ex))); + } + catch (Exception ex) + { + return new(ConversionResult.Failed(ex)); + } + } + } +} diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridStringArrayConverter.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridStringArrayConverter.cs new file mode 100644 index 000000000..e5bfe7099 --- /dev/null +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridStringArrayConverter.cs @@ -0,0 +1,64 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Converters; + +namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters +{ + /// + /// Converter to bind to string[] parameter. + /// + [SupportedConverterType(typeof(string[]))] + internal class EventGridStringArrayConverter : IInputConverter + { + public ValueTask ConvertAsync(ConverterContext context) + { + try + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + if (context.TargetType != typeof(string[])) + { + return new(ConversionResult.Unhandled()); + } + + if (context.Source is not string contextSource) + { + return new(ConversionResult.Failed(new InvalidOperationException("Context source must be a non-null string. Current type of context source is " + context?.Source?.GetType()))); + } + + var jsonData = JsonSerializer.Deserialize(contextSource, typeof(List)) as List; + List stringList = new List(); + + if (jsonData is not null) + { + return new(ConversionResult.Success(jsonData.Select(d => d?.ToString()).ToArray())); + } + } + catch (JsonException ex) + { + string msg = String.Format(CultureInfo.CurrentCulture, + @"Binding parameters to complex objects uses JSON serialization. + 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or + 2. Change the event payload to be valid json."); + + return new(ConversionResult.Failed(new InvalidOperationException(msg, ex))); + } + catch (Exception ex) + { + return new(ConversionResult.Failed(ex)); + } + + return new(ConversionResult.Unhandled()); + } + } +} diff --git a/extensions/Worker.Extensions.EventGrid/src/Worker.Extensions.EventGrid.csproj b/extensions/Worker.Extensions.EventGrid/src/Worker.Extensions.EventGrid.csproj index 735e95a33..2ea22e38d 100644 --- a/extensions/Worker.Extensions.EventGrid/src/Worker.Extensions.EventGrid.csproj +++ b/extensions/Worker.Extensions.EventGrid/src/Worker.Extensions.EventGrid.csproj @@ -6,7 +6,8 @@ Azure Event Grid extensions for .NET isolated functions - 3.2.1 + 3.3.0 + -preview1 false @@ -16,6 +17,11 @@ + + + + + \ No newline at end of file diff --git a/extensions/Worker.Extensions.EventGrid/src/release_notes.md b/extensions/Worker.Extensions.EventGrid/src/release_notes.md new file mode 100644 index 000000000..097c11925 --- /dev/null +++ b/extensions/Worker.Extensions.EventGrid/src/release_notes.md @@ -0,0 +1,9 @@ +## What's Changed + + + +### Microsoft.Azure.Functions.Worker.Extensions.EventGrid + +- Add ability to bind a event grid trigger to CloudEvent, CloudEvent[], EventGridEvent, EventGridEvent[], BinaryData, BinaryData[], and string[] diff --git a/samples/WorkerBindingSamples/EventGrid/EventGridTriggerBindingSamples.cs b/samples/WorkerBindingSamples/EventGrid/EventGridTriggerBindingSamples.cs new file mode 100644 index 000000000..5ffd9123a --- /dev/null +++ b/samples/WorkerBindingSamples/EventGrid/EventGridTriggerBindingSamples.cs @@ -0,0 +1,105 @@ +// Default URL for triggering event grid function in the local environment. +// http://localhost:7071/runtime/webhooks/EventGrid?functionName={functionname} +using System; +using Azure.Messaging; +using Azure.Messaging.EventGrid; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace WorkerBindingSamples.EventGrid +{ + public class EventGridTriggerBindingSamples + { + private readonly ILogger _logger; + + public EventGridTriggerBindingSamples(ILoggerFactory loggerFactory) + { + _logger = loggerFactory.CreateLogger(); + } + + [Function("MyEventFunction")] + public void MyEventFunction([EventGridTrigger] MyEvent input) + { + _logger.LogInformation(input.Data?.ToString()); + } + + [Function("CloudEventFunction")] + public void CloudEventFunction([EventGridTrigger] CloudEvent input) + { + _logger.LogInformation("Event type: {type}, Event subject: {subject}", input.Type, input.Subject); + } + + [Function("MultipleCloudEventFunction")] + public void MultipleCloudEventFunction([EventGridTrigger(IsBatched = true)] CloudEvent[] input) + { + for (var i = 0; i < input.Length; i++) + { + var cloudEvent = input[i]; + _logger.LogInformation("Event type: {type}, Event subject: {subject}", cloudEvent.Type, cloudEvent.Subject); + } + } + + [Function("EventGridEvent")] + public void EventGridEvent([EventGridTrigger] EventGridEvent input) + { + _logger.LogInformation("Event received: {event}", input.Data.ToString()); + } + + [Function("EventGridEventArray")] + public void EventGridEventArray([EventGridTrigger(IsBatched = true)] EventGridEvent[] input) + { + for (var i = 0; i < input.Length; i++) + { + var eventGridEvent = input[i]; + _logger.LogInformation("Event received: {event}", eventGridEvent.Data.ToString()); + } + } + + [Function("BinaryDataEvent")] + public void BinaryDataEvent([EventGridTrigger] BinaryData input) + { + _logger.LogInformation("Event received: {event}", input.ToString()); + } + + [Function("BinaryDataArrayEvent")] + public void BinaryDataArrayEvent([EventGridTrigger(IsBatched = true)] BinaryData[] input) + { + for (var i = 0; i < input.Length; i++) + { + var binaryDataEvent = input[i]; + _logger.LogInformation("Event received: {event}", binaryDataEvent.ToString()); + } + } + + [Function("StringArrayEvent")] + public void StringArrayEvent([EventGridTrigger(IsBatched = true)] string[] input) + { + for (var i = 0; i < input.Length; i++) + { + var stringEventGrid = input[i]; + _logger.LogInformation("Event received: {event}", stringEventGrid); + } + } + + [Function("StringEvent")] + public void StringEvent([EventGridTrigger] string input) + { + _logger.LogInformation("Event received: {event}", input); + } + } + + public class MyEvent + { + public string? Id { get; set; } + + public string? Topic { get; set; } + + public string? Subject { get; set; } + + public string? EventType { get; set; } + + public DateTime EventTime { get; set; } + + public object? Data { get; set; } + } +} diff --git a/samples/WorkerBindingSamples/WorkerBindingSamples.csproj b/samples/WorkerBindingSamples/WorkerBindingSamples.csproj index 5748826b7..460be44a7 100644 --- a/samples/WorkerBindingSamples/WorkerBindingSamples.csproj +++ b/samples/WorkerBindingSamples/WorkerBindingSamples.csproj @@ -13,6 +13,7 @@ + diff --git a/test/SdkE2ETests/PublishTests.cs b/test/SdkE2ETests/PublishTests.cs index b7d9feb45..6eeefa782 100644 --- a/test/SdkE2ETests/PublishTests.cs +++ b/test/SdkE2ETests/PublishTests.cs @@ -119,6 +119,9 @@ private async Task RunPublishTestForSdkTypeBindings(string outputDir, string add { extensions = new[] { + new Extension("EventGrid", + "Microsoft.Azure.WebJobs.Extensions.EventGrid.EventGridWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.EventGrid, Version=3.3.0.0, Culture=neutral, PublicKeyToken=014045d636e89289", + @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.EventGrid.dll"), new Extension("Startup", "Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.Startup, Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader, Version=1.0.0.0, Culture=neutral, PublicKeyToken=551316b6919f366c", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.dll"), diff --git a/test/Worker.Extensions.Tests/EventGrid/EventGridBinaryDataConverterTests.cs b/test/Worker.Extensions.Tests/EventGrid/EventGridBinaryDataConverterTests.cs new file mode 100644 index 000000000..30444150e --- /dev/null +++ b/test/Worker.Extensions.Tests/EventGrid/EventGridBinaryDataConverterTests.cs @@ -0,0 +1,96 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.EventGrid +{ + public class EventGridBinaryDataConverterTests + { + private EventGridBinaryDataConverter _eventGridConverter; + + public EventGridBinaryDataConverterTests() + { + _eventGridConverter = new EventGridBinaryDataConverter(); + } + + [Fact] + public async Task ConvertAsync_SourceAsObject_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(BinaryData), new object()); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_Returns_Success() + { + var context = new TestConverterContext(typeof(BinaryData), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is BinaryData); + } + + [Fact] + public async Task ConvertAsync_Returns_Unhandled_For_Unsupported_Type() + { + var context = new TestConverterContext(typeof(string), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + [Fact] + public async Task ConvertAsync_SourceAsObject_BinaryDataCollectible_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(BinaryData[]), new object()); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_BinaryDataCollectible_Returns_Success() + { + var context = new TestConverterContext(typeof(BinaryData[]), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is BinaryData[]); + Assert.Equal(2, ((BinaryData[])conversionResult.Value).Length); + } + + [Fact] + public async Task ConvertAsync_BinaryDataCollectible_Returns_Unhandled_For_Unsupported_Type() + { + var context = new TestConverterContext(typeof(string), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_SingleElement_Returns_Success() + { + var context = new TestConverterContext(typeof(BinaryData[]), "[{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is BinaryData[]); + Assert.Single((BinaryData[])conversionResult.Value); + } + } +} diff --git a/test/Worker.Extensions.Tests/EventGrid/EventGridCloudEventConverterTests.cs b/test/Worker.Extensions.Tests/EventGrid/EventGridCloudEventConverterTests.cs new file mode 100644 index 000000000..d3e8cf3ce --- /dev/null +++ b/test/Worker.Extensions.Tests/EventGrid/EventGridCloudEventConverterTests.cs @@ -0,0 +1,102 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Azure.Messaging; +using Azure.Messaging.EventGrid; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.Hosting; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.EventGrid +{ + public class EventGridCloudEventConverterTests + { + private EventGridCloudEventConverter _eventGridConverter; + + public EventGridCloudEventConverterTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + + _eventGridConverter = new EventGridCloudEventConverter(); + } + + [Fact] + public async Task ConvertAsync_SourceAsObject_ReturnsFailed() + { + var context = new TestConverterContext(typeof(CloudEvent), new object()); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_Returns_Success() + { + var context = new TestConverterContext(typeof(CloudEvent), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is CloudEvent); + } + + [Fact] + public async Task ConvertAsync_Returns_Unhandled_For_Unsupported_Type() + { + var context = new TestConverterContext(typeof(string), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_SourceAsObject_CloudEventCollectible_ReturnsFailed() + { + var context = new TestConverterContext(typeof(CloudEvent[]), new object()); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_CloudEventCollectible_Returns_Success() + { + var context = new TestConverterContext(typeof(CloudEvent[]), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is CloudEvent[]); + Assert.Equal(2, ((CloudEvent[])conversionResult.Value).Length); + } + + [Fact] + public async Task ConvertAsync_CloudEventCollectible_Returns_Unhandled_For_Unsupported_Type() + { + var context = new TestConverterContext(typeof(string), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_SingleElement_Returns_Success() + { + var context = new TestConverterContext(typeof(CloudEvent[]), "[{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Single((CloudEvent[])conversionResult.Value); + } + } +} diff --git a/test/Worker.Extensions.Tests/EventGrid/EventGridEventConverterTests.cs b/test/Worker.Extensions.Tests/EventGrid/EventGridEventConverterTests.cs new file mode 100644 index 000000000..aa1e5d475 --- /dev/null +++ b/test/Worker.Extensions.Tests/EventGrid/EventGridEventConverterTests.cs @@ -0,0 +1,110 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Azure.Messaging.EventGrid; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.Hosting; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.EventGrid +{ + public class EventGridEventConverterTests + { + private EventGridEventConverter _eventGridConverter; + + public EventGridEventConverterTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + + _eventGridConverter = new EventGridEventConverter(); + } + + [Fact] + public async Task ConvertAsync_SourceAsObject_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(EventGridEvent), new object()); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_Returns_Success() + { + var context = new TestConverterContext(typeof(EventGridEvent), "{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"test\",\"dataVersion\":\"test\"}"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is EventGridEvent); + } + + [Fact] + public async Task ConvertAsync_Returns_Unhandled_For_Unsupported_Type() + { + var context = new TestConverterContext(typeof(string), "{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"test\",\"dataVersion\":\"test\"}"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_Returns_Failed_Bad_Source() + { + var context = new TestConverterContext(typeof(EventGridEvent), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync__EventGridCollectible_SourceAsObject_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(EventGridEvent[]), new object()); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync__EventGridCollectible_Returns_Success() + { + var context = new TestConverterContext(typeof(EventGridEvent[]), "[{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"test\",\"dataVersion\":\"test\"},{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"lmao\",\"dataVersion\":\"test\"}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is EventGridEvent[]); + Assert.Equal(2, ((EventGridEvent[])conversionResult.Value).Length); + } + + [Fact] + public async Task ConvertAsync_EventGridCollectible_Returns_Unhandled_For_Unsupported_Type() + { + var context = new TestConverterContext(typeof(string), "{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"test\",\"dataVersion\":\"test\"}"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync__EventGridCollectible_Returns_Failed_Bad_Source() + { + var context = new TestConverterContext(typeof(EventGridEvent[]), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + } +} diff --git a/test/Worker.Extensions.Tests/EventGrid/EventGridStringArrayConverterTests.cs b/test/Worker.Extensions.Tests/EventGrid/EventGridStringArrayConverterTests.cs new file mode 100644 index 000000000..541d7647a --- /dev/null +++ b/test/Worker.Extensions.Tests/EventGrid/EventGridStringArrayConverterTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.EventGrid +{ + public class EventGridStringArrayConverterTests + { + private EventGridStringArrayConverter _eventGridConverter; + + public EventGridStringArrayConverterTests() + { + _eventGridConverter = new EventGridStringArrayConverter(); + } + + [Fact] + public async Task ConvertAsync_SourceAsObject_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(string[]), new object()); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_Returns_Success() + { + var context = new TestConverterContext(typeof(string[]), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is string[]); + Assert.Equal(2, ((string[])conversionResult.Value).Length); + } + + [Fact] + public async Task ConvertAsync_Returns_Unhandled_For_Unsupported_Type() + { + var context = new TestConverterContext(typeof(string), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_SingleElement_Returns_Success() + { + var context = new TestConverterContext(typeof(string[]), "[{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}]"); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is string[]); + Assert.Single((string[])conversionResult.Value); + } + } +} diff --git a/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj b/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj index 6818c4767..032918cdc 100644 --- a/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj +++ b/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj @@ -22,6 +22,7 @@ + From 67b642a42d3c444a463823b09112076c594b0696 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Thu, 13 Jul 2023 12:27:09 -0700 Subject: [PATCH 28/47] Refactor EventGrid extension converter, tests, and samples (#1695, #1716, #1727, #1723) --- .../src/EventGridTriggerAttribute.cs | 2 +- .../EventGridBinaryDataConverter.cs | 70 +++--------- .../EventGridCloudEventConverter.cs | 23 ++-- .../TypeConverters/EventGridConverterBase.cs | 50 +++++++++ .../TypeConverters/EventGridEventConverter.cs | 23 ++-- .../src/TypeConverters/EventGridHelper.cs | 42 ------- .../EventGridStringArrayConverter.cs | 51 ++------- .../EventGrid/CloudEventSamples.cs | 48 ++++++++ .../EventGrid/EventGridEventSamples.cs | 48 ++++++++ .../EventGridTriggerBindingSamples.cs | 105 ------------------ .../functions.metadata | 72 ++++++++++++ .../EventGridBinaryDataConverterTests.cs | 43 +++---- .../EventGridCloudEventConverterTests.cs | 46 +++----- .../EventGrid/EventGridEventConverterTests.cs | 54 +++------ .../EventGridStringArrayConverterTests.cs | 34 ++++-- .../EventGrid/EventGridTestHelper.cs | 54 +++++++++ 16 files changed, 397 insertions(+), 368 deletions(-) create mode 100644 extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridConverterBase.cs delete mode 100644 extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridHelper.cs create mode 100644 samples/WorkerBindingSamples/EventGrid/CloudEventSamples.cs create mode 100644 samples/WorkerBindingSamples/EventGrid/EventGridEventSamples.cs delete mode 100644 samples/WorkerBindingSamples/EventGrid/EventGridTriggerBindingSamples.cs create mode 100644 test/Worker.Extensions.Tests/EventGrid/EventGridTestHelper.cs diff --git a/extensions/Worker.Extensions.EventGrid/src/EventGridTriggerAttribute.cs b/extensions/Worker.Extensions.EventGrid/src/EventGridTriggerAttribute.cs index b0842fd0a..6dd90d720 100644 --- a/extensions/Worker.Extensions.EventGrid/src/EventGridTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventGrid/src/EventGridTriggerAttribute.cs @@ -7,11 +7,11 @@ namespace Microsoft.Azure.Functions.Worker { - [AllowConverterFallback(true)] [InputConverter(typeof(EventGridCloudEventConverter))] [InputConverter(typeof(EventGridEventConverter))] [InputConverter(typeof(EventGridBinaryDataConverter))] [InputConverter(typeof(EventGridStringArrayConverter))] + [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] public sealed class EventGridTriggerAttribute : TriggerBindingAttribute { private bool _isBatched = false; diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridBinaryDataConverter.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridBinaryDataConverter.cs index d55d9e28d..e504bbae3 100644 --- a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridBinaryDataConverter.cs +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridBinaryDataConverter.cs @@ -3,76 +3,42 @@ using System; using System.Collections.Generic; -using System.Globalization; +using System.Linq; using System.Text.Json; -using System.Threading.Tasks; using Microsoft.Azure.Functions.Worker.Converters; namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters { /// - /// Converter to bind to BinaryData or BinaryData[] parameter. + /// Converter to bind to or type parameters. /// - [SupportedConverterType(typeof(BinaryData))] - [SupportedConverterType(typeof(BinaryData[]))] - internal class EventGridBinaryDataConverter : IInputConverter + [SupportedTargetType(typeof(BinaryData))] + [SupportedTargetType(typeof(BinaryData[]))] + internal class EventGridBinaryDataConverter : EventGridConverterBase { - public ValueTask ConvertAsync(ConverterContext context) + protected override ConversionResult ConvertCore(Type targetType, string json) { - try + ConversionResult result = targetType switch { - if (context is null) - { - throw new ArgumentNullException(nameof(context)); - } + Type t when t == typeof(BinaryData) => ConversionResult.Success(BinaryData.FromString(json)), + Type t when t == typeof(BinaryData[]) => ConversionResult.Success(ConvertToBinaryDataArray(json)), + _ => ConversionResult.Failed(new InvalidOperationException($"'{targetType.Name}' is not supported by this converter.")) + }; - if (context.Source is not string contextSource) - { - return new(ConversionResult.Failed(new InvalidOperationException("Context source must be a non-null string. Current type of context source is " + context?.Source?.GetType()))); - } - - var targetType = context.TargetType; - - switch (targetType) - { - case Type t when t == typeof(BinaryData): - return new(ConversionResult.Success((BinaryData.FromString(contextSource)))); - case Type t when t == typeof(BinaryData[]): - return new(ConversionResult.Success(ConvertToBinaryDataArray(contextSource))); - } - } - catch (JsonException ex) - { - string msg = String.Format(CultureInfo.CurrentCulture, - @"Binding parameters to complex objects uses JSON serialization. - 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or - 2. Change the event payload to be valid json."); - - return new(ConversionResult.Failed(new InvalidOperationException(msg, ex))); - } - catch (Exception ex) - { - return new(ConversionResult.Failed(ex)); - } - - return new(ConversionResult.Unhandled()); + return result; } - private BinaryData?[]? ConvertToBinaryDataArray(string contextSource) + private BinaryData[] ConvertToBinaryDataArray(string json) { - var jsonData = JsonSerializer.Deserialize(contextSource, typeof(List)) as List; - List binaryDataList = new List(); + var data = JsonSerializer.Deserialize>(json); + var result = data.Select(item => BinaryData.FromString(item.ToString())).ToArray(); - if (jsonData is not null) + if (result is null) { - foreach (var item in jsonData) - { - var binaryData = item == null? null: BinaryData.FromString(item.ToString()); - binaryDataList.Add(binaryData); - } + throw new Exception("Unable to convert to BinaryData[]."); } - return binaryDataList.ToArray(); + return result; } } } diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridCloudEventConverter.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridCloudEventConverter.cs index 67e4e3eab..6ce7155b8 100644 --- a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridCloudEventConverter.cs +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridCloudEventConverter.cs @@ -2,32 +2,29 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; -using System.Threading.Tasks; +using System.Text.Json; using Azure.Messaging; using Microsoft.Azure.Functions.Worker.Converters; namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters { /// - /// Converter to bind to CloudEvent or CloudEvent[] parameter. + /// Converter to bind to or type parameters. /// - [SupportedConverterType(typeof(CloudEvent))] - [SupportedConverterType(typeof(CloudEvent[]))] - internal class EventGridCloudEventConverter: IInputConverter + [SupportedTargetType(typeof(CloudEvent))] + [SupportedTargetType(typeof(CloudEvent[]))] + internal class EventGridCloudEventConverter: EventGridConverterBase { - public ValueTask ConvertAsync(ConverterContext context) + protected override ConversionResult ConvertCore(Type targetType, string json) { - if (context is null) - { - return new(ConversionResult.Failed(new ArgumentNullException(nameof(context)))); - } + var cloudEvent = JsonSerializer.Deserialize(json, targetType); - if (context.TargetType != typeof(CloudEvent) && context.TargetType != typeof(CloudEvent[])) + if (cloudEvent is null) { - return new(ConversionResult.Unhandled()); + return ConversionResult.Failed(new Exception("Unable to convert to CloudEvent.")); } - return EventGridHelper.DeserializeToTargetType(context); + return ConversionResult.Success(cloudEvent); } } } diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridConverterBase.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridConverterBase.cs new file mode 100644 index 000000000..7d4eeb6f1 --- /dev/null +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridConverterBase.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Converters; + +namespace Microsoft.Azure.Functions.Worker +{ + internal abstract class EventGridConverterBase : IInputConverter + { + public EventGridConverterBase() { } + + public ValueTask ConvertAsync(ConverterContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + try + { + if (context.Source is not string json) + { + throw new InvalidOperationException("Context source must be a non-null string"); + } + + var result = ConvertCore(context.TargetType, json); + return new ValueTask(result); + } + catch (JsonException ex) + { + string msg = String.Format(CultureInfo.CurrentCulture, + @"Binding parameters to complex objects uses JSON serialization. + 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or + 2. Change the queue payload to be valid json."); + + return new ValueTask(ConversionResult.Failed(new InvalidOperationException(msg, ex))); + } + catch (Exception ex) + { + return new ValueTask(ConversionResult.Failed(ex)); + } + } + + protected abstract ConversionResult ConvertCore(Type targetType, string json); + } +} diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridEventConverter.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridEventConverter.cs index 7397d2278..9cf023f7f 100644 --- a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridEventConverter.cs +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridEventConverter.cs @@ -2,32 +2,29 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; -using System.Threading.Tasks; +using System.Text.Json; using Azure.Messaging.EventGrid; using Microsoft.Azure.Functions.Worker.Converters; namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters { /// - /// Converter to bind to EventGridEvent or EventGridEvent[] parameter. + /// Converter to bind to or type parameters. /// - [SupportedConverterType(typeof(EventGridEvent))] - [SupportedConverterType(typeof(EventGridEvent[]))] - internal class EventGridEventConverter : IInputConverter + [SupportedTargetType(typeof(EventGridEvent))] + [SupportedTargetType(typeof(EventGridEvent[]))] + internal class EventGridEventConverter : EventGridConverterBase { - public ValueTask ConvertAsync(ConverterContext context) + protected override ConversionResult ConvertCore(Type targetType, string json) { - if (context is null) - { - return new(ConversionResult.Failed(new ArgumentNullException(nameof(context)))); - } + var eventGridEvent = JsonSerializer.Deserialize(json, targetType); - if (context?.TargetType != typeof(EventGridEvent) && context?.TargetType != typeof(EventGridEvent[])) + if (eventGridEvent is null) { - return new(ConversionResult.Unhandled()); + return ConversionResult.Failed(new Exception("Unable to convert to EventGridEvent.")); } - return EventGridHelper.DeserializeToTargetType(context); + return ConversionResult.Success(eventGridEvent); } } } diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridHelper.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridHelper.cs deleted file mode 100644 index 9347e21f3..000000000 --- a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridHelper.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Globalization; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.Azure.Functions.Worker.Converters; - -namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters -{ - internal static class EventGridHelper - { - internal static ValueTask DeserializeToTargetType(ConverterContext context) - { - try - { - if (context.Source is not string contextSource) - { - return new(ConversionResult.Failed(new InvalidOperationException("Context source must be a non-null string. Current type of context source is " + context?.Source?.GetType()))); - } - - var targetType = context.TargetType; - var item = JsonSerializer.Deserialize(contextSource, targetType); - return new(ConversionResult.Success(item)); - } - catch (JsonException ex) - { - string msg = String.Format(CultureInfo.CurrentCulture, - @"Binding parameters to complex objects uses JSON serialization. - 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or - 2. Change the event payload to be valid json."); - - return new(ConversionResult.Failed(new InvalidOperationException(msg, ex))); - } - catch (Exception ex) - { - return new(ConversionResult.Failed(ex)); - } - } - } -} diff --git a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridStringArrayConverter.cs b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridStringArrayConverter.cs index e5bfe7099..18b0688ac 100644 --- a/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridStringArrayConverter.cs +++ b/extensions/Worker.Extensions.EventGrid/src/TypeConverters/EventGridStringArrayConverter.cs @@ -3,62 +3,29 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Text.Json; -using System.Threading.Tasks; using Microsoft.Azure.Functions.Worker.Converters; namespace Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters { /// - /// Converter to bind to string[] parameter. + /// Converter to bind to type parameters. /// - [SupportedConverterType(typeof(string[]))] - internal class EventGridStringArrayConverter : IInputConverter + [SupportedTargetType(typeof(string[]))] + internal class EventGridStringArrayConverter : EventGridConverterBase { - public ValueTask ConvertAsync(ConverterContext context) + protected override ConversionResult ConvertCore(Type targetType, string json) { - try - { - if (context is null) - { - throw new ArgumentNullException(nameof(context)); - } - - if (context.TargetType != typeof(string[])) - { - return new(ConversionResult.Unhandled()); - } - - if (context.Source is not string contextSource) - { - return new(ConversionResult.Failed(new InvalidOperationException("Context source must be a non-null string. Current type of context source is " + context?.Source?.GetType()))); - } - - var jsonData = JsonSerializer.Deserialize(contextSource, typeof(List)) as List; - List stringList = new List(); + var jsonData = JsonSerializer.Deserialize>(json); + var result = jsonData.Select(d => d.ToString()).ToArray(); - if (jsonData is not null) - { - return new(ConversionResult.Success(jsonData.Select(d => d?.ToString()).ToArray())); - } - } - catch (JsonException ex) - { - string msg = String.Format(CultureInfo.CurrentCulture, - @"Binding parameters to complex objects uses JSON serialization. - 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or - 2. Change the event payload to be valid json."); - - return new(ConversionResult.Failed(new InvalidOperationException(msg, ex))); - } - catch (Exception ex) + if (result is null) { - return new(ConversionResult.Failed(ex)); + return ConversionResult.Failed(new Exception("Unable to convert to string[].")); } - return new(ConversionResult.Unhandled()); + return ConversionResult.Success(result); } } } diff --git a/samples/WorkerBindingSamples/EventGrid/CloudEventSamples.cs b/samples/WorkerBindingSamples/EventGrid/CloudEventSamples.cs new file mode 100644 index 000000000..1f3088bca --- /dev/null +++ b/samples/WorkerBindingSamples/EventGrid/CloudEventSamples.cs @@ -0,0 +1,48 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +// Default URL for triggering event grid function in the local environment. +// http://localhost:7071/runtime/webhooks/EventGrid?functionName={functionname} + +using Azure.Messaging; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace SampleApp +{ + /// + /// Samples demonstrating binding to the type. + /// + public class CloudEventSamples + { + private readonly ILogger _logger; + + public CloudEventSamples(ILogger logger) + { + _logger = logger; + } + + /// + /// This function demonstrates binding to a single . + /// + [Function(nameof(CloudEventFunction))] + public void CloudEventFunction([EventGridTrigger] CloudEvent cloudEvent) + { + _logger.LogInformation("Event type: {type}, Event subject: {subject}", cloudEvent.Type, cloudEvent.Subject); + } + + /// + /// This function demonstrates binding to an array of . + /// Note that when doing so, you must also set the property + /// to true. + /// + [Function(nameof(CloudEventBatchFunction))] + public void CloudEventBatchFunction([EventGridTrigger(IsBatched = true)] CloudEvent[] cloudEvents) + { + foreach (var cloudEvent in cloudEvents) + { + _logger.LogInformation("Event type: {type}, Event subject: {subject}", cloudEvent.Type, cloudEvent.Subject); + } + } + } +} diff --git a/samples/WorkerBindingSamples/EventGrid/EventGridEventSamples.cs b/samples/WorkerBindingSamples/EventGrid/EventGridEventSamples.cs new file mode 100644 index 000000000..024b60c36 --- /dev/null +++ b/samples/WorkerBindingSamples/EventGrid/EventGridEventSamples.cs @@ -0,0 +1,48 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +// Default URL for triggering event grid function in the local environment. +// http://localhost:7071/runtime/webhooks/EventGrid?functionName={functionname} + +using Azure.Messaging.EventGrid; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace SampleApp +{ + /// + /// Samples demonstrating binding to the type. + /// + public class EventGridEventSamples + { + private readonly ILogger _logger; + + public EventGridEventSamples(ILogger logger) + { + _logger = logger; + } + + /// + /// This function demonstrates binding to a single . + /// + [Function(nameof(EventGridEventFunction))] + public void EventGridEventFunction([EventGridTrigger] EventGridEvent eventGridEvent) + { + _logger.LogInformation("Event received: {event}", eventGridEvent.Data.ToString()); + } + + /// + /// This function demonstrates binding to an array of . + /// Note that when doing so, you must also set the property + /// to true. + /// + [Function(nameof(EventGridEventBatchFunction))] + public void EventGridEventBatchFunction([EventGridTrigger(IsBatched = true)] EventGridEvent[] eventGridEvents) + { + foreach (var eventGridEvent in eventGridEvents) + { + _logger.LogInformation("Event received: {event}", eventGridEvent.Data.ToString()); + } + } + } +} diff --git a/samples/WorkerBindingSamples/EventGrid/EventGridTriggerBindingSamples.cs b/samples/WorkerBindingSamples/EventGrid/EventGridTriggerBindingSamples.cs deleted file mode 100644 index 5ffd9123a..000000000 --- a/samples/WorkerBindingSamples/EventGrid/EventGridTriggerBindingSamples.cs +++ /dev/null @@ -1,105 +0,0 @@ -// Default URL for triggering event grid function in the local environment. -// http://localhost:7071/runtime/webhooks/EventGrid?functionName={functionname} -using System; -using Azure.Messaging; -using Azure.Messaging.EventGrid; -using Microsoft.Azure.Functions.Worker; -using Microsoft.Extensions.Logging; - -namespace WorkerBindingSamples.EventGrid -{ - public class EventGridTriggerBindingSamples - { - private readonly ILogger _logger; - - public EventGridTriggerBindingSamples(ILoggerFactory loggerFactory) - { - _logger = loggerFactory.CreateLogger(); - } - - [Function("MyEventFunction")] - public void MyEventFunction([EventGridTrigger] MyEvent input) - { - _logger.LogInformation(input.Data?.ToString()); - } - - [Function("CloudEventFunction")] - public void CloudEventFunction([EventGridTrigger] CloudEvent input) - { - _logger.LogInformation("Event type: {type}, Event subject: {subject}", input.Type, input.Subject); - } - - [Function("MultipleCloudEventFunction")] - public void MultipleCloudEventFunction([EventGridTrigger(IsBatched = true)] CloudEvent[] input) - { - for (var i = 0; i < input.Length; i++) - { - var cloudEvent = input[i]; - _logger.LogInformation("Event type: {type}, Event subject: {subject}", cloudEvent.Type, cloudEvent.Subject); - } - } - - [Function("EventGridEvent")] - public void EventGridEvent([EventGridTrigger] EventGridEvent input) - { - _logger.LogInformation("Event received: {event}", input.Data.ToString()); - } - - [Function("EventGridEventArray")] - public void EventGridEventArray([EventGridTrigger(IsBatched = true)] EventGridEvent[] input) - { - for (var i = 0; i < input.Length; i++) - { - var eventGridEvent = input[i]; - _logger.LogInformation("Event received: {event}", eventGridEvent.Data.ToString()); - } - } - - [Function("BinaryDataEvent")] - public void BinaryDataEvent([EventGridTrigger] BinaryData input) - { - _logger.LogInformation("Event received: {event}", input.ToString()); - } - - [Function("BinaryDataArrayEvent")] - public void BinaryDataArrayEvent([EventGridTrigger(IsBatched = true)] BinaryData[] input) - { - for (var i = 0; i < input.Length; i++) - { - var binaryDataEvent = input[i]; - _logger.LogInformation("Event received: {event}", binaryDataEvent.ToString()); - } - } - - [Function("StringArrayEvent")] - public void StringArrayEvent([EventGridTrigger(IsBatched = true)] string[] input) - { - for (var i = 0; i < input.Length; i++) - { - var stringEventGrid = input[i]; - _logger.LogInformation("Event received: {event}", stringEventGrid); - } - } - - [Function("StringEvent")] - public void StringEvent([EventGridTrigger] string input) - { - _logger.LogInformation("Event received: {event}", input); - } - } - - public class MyEvent - { - public string? Id { get; set; } - - public string? Topic { get; set; } - - public string? Subject { get; set; } - - public string? EventType { get; set; } - - public DateTime EventTime { get; set; } - - public object? Data { get; set; } - } -} diff --git a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata index 87adc894c..abc769bcc 100644 --- a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata +++ b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata @@ -314,5 +314,77 @@ } } ] + }, + { + "name": "CloudEventFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CloudEventSamples.CloudEventFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "cloudEvent", + "direction": "In", + "type": "eventGridTrigger", + "cardinality": "One", + "properties": {} + } + ] + }, + { + "name": "CloudEventBatchFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CloudEventSamples.CloudEventBatchFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "cloudEvents", + "direction": "In", + "type": "eventGridTrigger", + "cardinality": "Many", + "properties": {} + } + ] + }, + { + "name": "EventGridEventFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.EventGridEventSamples.EventGridEventFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "eventGridEvent", + "direction": "In", + "type": "eventGridTrigger", + "cardinality": "One", + "properties": {} + } + ] + }, + { + "name": "EventGridEventBatchFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.EventGridEventSamples.EventGridEventBatchFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "eventGridEvents", + "direction": "In", + "type": "eventGridTrigger", + "cardinality": "Many", + "properties": {} + } + ] } ] \ No newline at end of file diff --git a/test/Worker.Extensions.Tests/EventGrid/EventGridBinaryDataConverterTests.cs b/test/Worker.Extensions.Tests/EventGrid/EventGridBinaryDataConverterTests.cs index 30444150e..0d7ad9f97 100644 --- a/test/Worker.Extensions.Tests/EventGrid/EventGridBinaryDataConverterTests.cs +++ b/test/Worker.Extensions.Tests/EventGrid/EventGridBinaryDataConverterTests.cs @@ -20,71 +20,64 @@ public EventGridBinaryDataConverterTests() } [Fact] - public async Task ConvertAsync_SourceAsObject_ReturnsUnhandled() + public async Task ConvertAsync_Source_IsNotAString_ReturnsFailed() { var context = new TestConverterContext(typeof(BinaryData), new object()); var conversionResult = await _eventGridConverter.ConvertAsync(context); Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Context source must be a non-null string", conversionResult.Error.Message); } [Fact] - public async Task ConvertAsync_Returns_Success() + public async Task ConvertAsync_UnsupportedTargetType_ReturnsFailed() { - var context = new TestConverterContext(typeof(BinaryData), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + var context = new TestConverterContext(typeof(string), ""); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.True(conversionResult.Value is BinaryData); + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); } [Fact] - public async Task ConvertAsync_Returns_Unhandled_For_Unsupported_Type() + public async Task ConvertAsync_InvalidJson_ThrowsJsonException_ReturnsFailed() { - var context = new TestConverterContext(typeof(string), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); - - var conversionResult = await _eventGridConverter.ConvertAsync(context); - - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); - } - [Fact] - public async Task ConvertAsync_SourceAsObject_BinaryDataCollectible_ReturnsUnhandled() - { - var context = new TestConverterContext(typeof(BinaryData[]), new object()); + var context = new TestConverterContext(typeof(BinaryData[]), @"{""invalid"" :json""}"); var conversionResult = await _eventGridConverter.ConvertAsync(context); Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); } [Fact] - public async Task ConvertAsync_BinaryDataCollectible_Returns_Success() + public async Task ConvertAsync_SingleBinaryData_ReturnsSuccess() { - var context = new TestConverterContext(typeof(BinaryData[]), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + var context = new TestConverterContext(typeof(BinaryData), EventGridTestHelper.GetEventGridJsonData()); var conversionResult = await _eventGridConverter.ConvertAsync(context); Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.True(conversionResult.Value is BinaryData[]); - Assert.Equal(2, ((BinaryData[])conversionResult.Value).Length); + Assert.True(conversionResult.Value is BinaryData); } [Fact] - public async Task ConvertAsync_BinaryDataCollectible_Returns_Unhandled_For_Unsupported_Type() + public async Task ConvertAsync_BinaryDataArray_ReturnsSuccess() { - var context = new TestConverterContext(typeof(string), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + var context = new TestConverterContext(typeof(BinaryData[]), EventGridTestHelper.GetEventGridJsonDataArray()); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is BinaryData[]); + Assert.Equal(2, ((BinaryData[])conversionResult.Value).Length); } [Fact] - public async Task ConvertAsync_SingleElement_Returns_Success() + public async Task ConvertAsync_BinaryDataArray_SingleElement_ReturnsSuccess() { - var context = new TestConverterContext(typeof(BinaryData[]), "[{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}]"); + var context = new TestConverterContext(typeof(BinaryData[]), $"[{EventGridTestHelper.GetEventGridJsonData()}]"); var conversionResult = await _eventGridConverter.ConvertAsync(context); diff --git a/test/Worker.Extensions.Tests/EventGrid/EventGridCloudEventConverterTests.cs b/test/Worker.Extensions.Tests/EventGrid/EventGridCloudEventConverterTests.cs index d3e8cf3ce..b0f944c7b 100644 --- a/test/Worker.Extensions.Tests/EventGrid/EventGridCloudEventConverterTests.cs +++ b/test/Worker.Extensions.Tests/EventGrid/EventGridCloudEventConverterTests.cs @@ -1,10 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; using System.Threading.Tasks; using Azure.Messaging; -using Azure.Messaging.EventGrid; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Extensions.EventGrid.TypeConverters; @@ -21,55 +19,57 @@ public class EventGridCloudEventConverterTests public EventGridCloudEventConverterTests() { var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); - _eventGridConverter = new EventGridCloudEventConverter(); } [Fact] - public async Task ConvertAsync_SourceAsObject_ReturnsFailed() + public async Task ConvertAsync_Source_IsNotAString_ReturnsFailed() { var context = new TestConverterContext(typeof(CloudEvent), new object()); var conversionResult = await _eventGridConverter.ConvertAsync(context); Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Context source must be a non-null string", conversionResult.Error.Message); } [Fact] - public async Task ConvertAsync_Returns_Success() + public async Task ConvertAsync_UnsupportedTargetType_ReturnsFailed() { - var context = new TestConverterContext(typeof(CloudEvent), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + var context = new TestConverterContext(typeof(string), ""); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.True(conversionResult.Value is CloudEvent); + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); } [Fact] - public async Task ConvertAsync_Returns_Unhandled_For_Unsupported_Type() + public async Task ConvertAsync_InvalidJson_ThrowsJsonException_ReturnsFailed() { - var context = new TestConverterContext(typeof(string), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + var context = new TestConverterContext(typeof(CloudEvent[]), @"{""invalid"" :json""}"); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); } [Fact] - public async Task ConvertAsync_SourceAsObject_CloudEventCollectible_ReturnsFailed() + public async Task ConvertAsync_SingleCloudEvent_ReturnsSuccess() { - var context = new TestConverterContext(typeof(CloudEvent[]), new object()); + var context = new TestConverterContext(typeof(CloudEvent), EventGridTestHelper.GetEventGridJsonData()); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is CloudEvent); } + [Fact] - public async Task ConvertAsync_CloudEventCollectible_Returns_Success() + public async Task ConvertAsync_CloudEventArray_ReturnsSuccess() { - var context = new TestConverterContext(typeof(CloudEvent[]), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + var context = new TestConverterContext(typeof(CloudEvent[]), EventGridTestHelper.GetEventGridJsonDataArray()); var conversionResult = await _eventGridConverter.ConvertAsync(context); @@ -79,19 +79,9 @@ public async Task ConvertAsync_CloudEventCollectible_Returns_Success() } [Fact] - public async Task ConvertAsync_CloudEventCollectible_Returns_Unhandled_For_Unsupported_Type() - { - var context = new TestConverterContext(typeof(string), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); - - var conversionResult = await _eventGridConverter.ConvertAsync(context); - - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); - } - - [Fact] - public async Task ConvertAsync_SingleElement_Returns_Success() + public async Task ConvertAsync_CloudEventArray_SingleElement_ReturnsSuccess() { - var context = new TestConverterContext(typeof(CloudEvent[]), "[{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}]"); + var context = new TestConverterContext(typeof(CloudEvent[]), $"[{EventGridTestHelper.GetEventGridJsonData()}]"); var conversionResult = await _eventGridConverter.ConvertAsync(context); diff --git a/test/Worker.Extensions.Tests/EventGrid/EventGridEventConverterTests.cs b/test/Worker.Extensions.Tests/EventGrid/EventGridEventConverterTests.cs index aa1e5d475..054059721 100644 --- a/test/Worker.Extensions.Tests/EventGrid/EventGridEventConverterTests.cs +++ b/test/Worker.Extensions.Tests/EventGrid/EventGridEventConverterTests.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; using System.Threading.Tasks; using Azure.Messaging.EventGrid; using Microsoft.Azure.Functions.Worker; @@ -25,60 +24,63 @@ public EventGridEventConverterTests() } [Fact] - public async Task ConvertAsync_SourceAsObject_ReturnsUnhandled() + public async Task ConvertAsync_Source_IsNotAString_ReturnsUnhandled() { var context = new TestConverterContext(typeof(EventGridEvent), new object()); var conversionResult = await _eventGridConverter.ConvertAsync(context); Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Context source must be a non-null string", conversionResult.Error.Message); } [Fact] - public async Task ConvertAsync_Returns_Success() + public async Task ConvertAsync_UnsupportedTargetType_ReturnsFailed() { - var context = new TestConverterContext(typeof(EventGridEvent), "{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"test\",\"dataVersion\":\"test\"}"); + var context = new TestConverterContext(typeof(string), ""); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.True(conversionResult.Value is EventGridEvent); + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); } [Fact] - public async Task ConvertAsync_Returns_Unhandled_For_Unsupported_Type() + public async Task ConvertAsync_InvalidJson_ThrowsJsonException_ReturnsFailed() { - var context = new TestConverterContext(typeof(string), "{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"test\",\"dataVersion\":\"test\"}"); + var context = new TestConverterContext(typeof(EventGridEvent), @"{""invalid"" :json""}"); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); } [Fact] - public async Task ConvertAsync_Returns_Failed_Bad_Source() + public async Task ConvertAsync_InvalidData_Throws_ReturnsFailed() { - var context = new TestConverterContext(typeof(EventGridEvent), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); + var context = new TestConverterContext(typeof(EventGridEvent), EventGridTestHelper.GetEventGridJsonData()); var conversionResult = await _eventGridConverter.ConvertAsync(context); Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Value cannot be null. (Parameter 'EventType')", conversionResult.Error.Message); } [Fact] - public async Task ConvertAsync__EventGridCollectible_SourceAsObject_ReturnsUnhandled() + public async Task ConvertAsync_SingleEventGridEvent_ReturnsSuccess() { - var context = new TestConverterContext(typeof(EventGridEvent[]), new object()); + var context = new TestConverterContext(typeof(EventGridEvent), EventGridTestHelper.GetEventGridEventJsonData()); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is EventGridEvent); } [Fact] - public async Task ConvertAsync__EventGridCollectible_Returns_Success() + public async Task ConvertAsync_EventGridArray_ReturnsSuccess() { - var context = new TestConverterContext(typeof(EventGridEvent[]), "[{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"test\",\"dataVersion\":\"test\"},{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"lmao\",\"dataVersion\":\"test\"}]"); + var context = new TestConverterContext(typeof(EventGridEvent[]), EventGridTestHelper.GetEventGridEventJsonDataArray()); var conversionResult = await _eventGridConverter.ConvertAsync(context); @@ -86,25 +88,5 @@ public async Task ConvertAsync__EventGridCollectible_Returns_Success() Assert.True(conversionResult.Value is EventGridEvent[]); Assert.Equal(2, ((EventGridEvent[])conversionResult.Value).Length); } - - [Fact] - public async Task ConvertAsync_EventGridCollectible_Returns_Unhandled_For_Unsupported_Type() - { - var context = new TestConverterContext(typeof(string), "{\"id\":\"'1\",\"topic\":\"hello\",\"subject\":\"yoursubject\",\"eventType\":\"yourEventType\",\"eventTime\":\"2018-01-23T17:02:19.6069787Z\",\"data\":\"test\",\"dataVersion\":\"test\"}"); - - var conversionResult = await _eventGridConverter.ConvertAsync(context); - - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); - } - - [Fact] - public async Task ConvertAsync__EventGridCollectible_Returns_Failed_Bad_Source() - { - var context = new TestConverterContext(typeof(EventGridEvent[]), "{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}"); - - var conversionResult = await _eventGridConverter.ConvertAsync(context); - - Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - } } } diff --git a/test/Worker.Extensions.Tests/EventGrid/EventGridStringArrayConverterTests.cs b/test/Worker.Extensions.Tests/EventGrid/EventGridStringArrayConverterTests.cs index 541d7647a..491f5ee6f 100644 --- a/test/Worker.Extensions.Tests/EventGrid/EventGridStringArrayConverterTests.cs +++ b/test/Worker.Extensions.Tests/EventGrid/EventGridStringArrayConverterTests.cs @@ -19,41 +19,53 @@ public EventGridStringArrayConverterTests() } [Fact] - public async Task ConvertAsync_SourceAsObject_ReturnsUnhandled() + public async Task ConvertAsync_Source_IsNotAString_ReturnsUnhandled() { var context = new TestConverterContext(typeof(string[]), new object()); var conversionResult = await _eventGridConverter.ConvertAsync(context); Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Context source must be a non-null string", conversionResult.Error.Message); } [Fact] - public async Task ConvertAsync_Returns_Success() + public async Task ConvertAsync_UnsupportedTargetType_ReturnsFailed() { - var context = new TestConverterContext(typeof(string[]), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + var context = new TestConverterContext(typeof(byte[]), ""); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); - Assert.True(conversionResult.Value is string[]); - Assert.Equal(2, ((string[])conversionResult.Value).Length); + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); } [Fact] - public async Task ConvertAsync_Returns_Unhandled_For_Unsupported_Type() + public async Task ConvertAsync_InvalidJson_ThrowsJsonException_ReturnsFailed() { - var context = new TestConverterContext(typeof(string), "[{\"specversion\":\"1.0\",\"id\":\"b85d631a-101e-005a-02f2-cee7aa06f148\",\"type\":\"zohan.music.request\",\"source\":\"https://zohan.dev/music/\",\"subject\":\"zohan/music/requests/4322\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"Gerardo\",\"song\":\"Rico Suave\"}},{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"life is very lit\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"life is lit\"}}]"); + var context = new TestConverterContext(typeof(string[]), @"{""invalid"" :json""}"); var conversionResult = await _eventGridConverter.ConvertAsync(context); - Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Contains("Binding parameters to complex objects uses JSON serialization", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_StringArray_ReturnsSuccess() + { + var context = new TestConverterContext(typeof(string[]), EventGridTestHelper.GetEventGridJsonDataArray()); + + var conversionResult = await _eventGridConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.True(conversionResult.Value is string[]); + Assert.Equal(2, ((string[])conversionResult.Value).Length); } [Fact] - public async Task ConvertAsync_SingleElement_Returns_Success() + public async Task ConvertAsync_StringArray_SingleElement_Returns_Success() { - var context = new TestConverterContext(typeof(string[]), "[{\"specversion\":\"1.0\",\"id\":\"2947780a-356b-c5a5-feb4-f5261fb2f155\",\"type\":\"test\",\"source\":\"moo\",\"subject\":\"lol test\",\"time\":\"2020-09-14T10:00:00Z\",\"data\":{\"artist\":\"wooo\",\"song\":\"some song\"}}]"); + var context = new TestConverterContext(typeof(string[]), $"[{EventGridTestHelper.GetEventGridJsonData()}]"); var conversionResult = await _eventGridConverter.ConvertAsync(context); diff --git a/test/Worker.Extensions.Tests/EventGrid/EventGridTestHelper.cs b/test/Worker.Extensions.Tests/EventGrid/EventGridTestHelper.cs new file mode 100644 index 000000000..9ab8fa600 --- /dev/null +++ b/test/Worker.Extensions.Tests/EventGrid/EventGridTestHelper.cs @@ -0,0 +1,54 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Google.Protobuf; +using Microsoft.Azure.Functions.Worker.Grpc.Messages; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests +{ + internal static class EventGridTestHelper + { + public static string GetEventGridJsonData(string id = "2947780a-356b-c5a5-feb4-f5261fb2f155", string song = "Vampire") + { + return $@"{{ + ""specversion"" : ""1.0"", + ""id"" : ""{id}"", + ""type"" : ""UnitTestData"", + ""source"" : ""UnitTest"", + ""subject"" : ""Song"", + ""time"" : ""2020-09-14T10:00:00Z"", + ""data"" : {{ ""artist"":""Olivia Rodrigo"",""song"":""{song}"" }} + }}"; + } + + public static string GetEventGridJsonDataArray() + { + return $@"[ + {GetEventGridJsonData("2947780a-356b-c5a5-feb4-f5261fb2f155", "Driver's License")}, + {GetEventGridJsonData("b85d631a-101e-005a-02f2-cee7aa06f148", "Deja Vu")} + ]"; + } + + public static string GetEventGridEventJsonData(string id = "2947780a-356b-c5a5-feb4-f5261fb2f155", string song = "Vampire") + { + return $@"{{ + ""id"" : ""{id}"", + ""topic"" : ""UnitTestData"", + ""subject"" : ""Song"", + ""eventType"" : ""MyEvent"", + ""eventTime"" : ""2020-09-14T10:00:00Z"", + ""data"" : {{ ""artist"":""Olivia Rodrigo"",""song"":""{song}"" }}, + ""dataVersion"" : ""1.0"" + }}"; + } + + public static string GetEventGridEventJsonDataArray() + { + return $@"[ + {GetEventGridEventJsonData("2947780a-356b-c5a5-feb4-f5261fb2f155", "Driver's License")}, + {GetEventGridEventJsonData("b85d631a-101e-005a-02f2-cee7aa06f148", "Deja Vu")} + ]"; + } + } +} \ No newline at end of file From 7d39c35ba8e27dda27ca17dda612eba5fd9e28d7 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Thu, 13 Jul 2023 12:28:06 -0700 Subject: [PATCH 29/47] Prepare EventGrid extension for release (#1747) --- extensions/Worker.Extensions.EventGrid/release_notes.md | 4 ++-- .../src/Worker.Extensions.EventGrid.csproj | 1 - .../Worker.Extensions.EventGrid/src/release_notes.md | 9 --------- 3 files changed, 2 insertions(+), 12 deletions(-) delete mode 100644 extensions/Worker.Extensions.EventGrid/src/release_notes.md diff --git a/extensions/Worker.Extensions.EventGrid/release_notes.md b/extensions/Worker.Extensions.EventGrid/release_notes.md index 9e82bcaf3..8113b1628 100644 --- a/extensions/Worker.Extensions.EventGrid/release_notes.md +++ b/extensions/Worker.Extensions.EventGrid/release_notes.md @@ -4,6 +4,6 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Extensions.EventGrid +### Microsoft.Azure.Functions.Worker.Extensions.EventGrid 3.3.0 -- +- Add ability to bind a event grid trigger to CloudEvent, CloudEvent[], EventGridEvent, EventGridEvent[], BinaryData, BinaryData[], and string[] diff --git a/extensions/Worker.Extensions.EventGrid/src/Worker.Extensions.EventGrid.csproj b/extensions/Worker.Extensions.EventGrid/src/Worker.Extensions.EventGrid.csproj index 2ea22e38d..476552aec 100644 --- a/extensions/Worker.Extensions.EventGrid/src/Worker.Extensions.EventGrid.csproj +++ b/extensions/Worker.Extensions.EventGrid/src/Worker.Extensions.EventGrid.csproj @@ -7,7 +7,6 @@ 3.3.0 - -preview1 false diff --git a/extensions/Worker.Extensions.EventGrid/src/release_notes.md b/extensions/Worker.Extensions.EventGrid/src/release_notes.md deleted file mode 100644 index 097c11925..000000000 --- a/extensions/Worker.Extensions.EventGrid/src/release_notes.md +++ /dev/null @@ -1,9 +0,0 @@ -## What's Changed - - - -### Microsoft.Azure.Functions.Worker.Extensions.EventGrid - -- Add ability to bind a event grid trigger to CloudEvent, CloudEvent[], EventGridEvent, EventGridEvent[], BinaryData, BinaryData[], and string[] From 26d9458ebd0f639cae692b641a8d2a34bf4c90aa Mon Sep 17 00:00:00 2001 From: sarah <35204912+satvu@users.noreply.github.com> Date: Fri, 21 Jul 2023 11:33:54 -0700 Subject: [PATCH 30/47] Update to protobuf v1.10.0 (#1774) --- .../src/proto/FunctionRpc.proto | 11 +++++++++++ release_notes.md | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/protos/azure-functions-language-worker-protobuf/src/proto/FunctionRpc.proto b/protos/azure-functions-language-worker-protobuf/src/proto/FunctionRpc.proto index b34ac8274..64162e12d 100644 --- a/protos/azure-functions-language-worker-protobuf/src/proto/FunctionRpc.proto +++ b/protos/azure-functions-language-worker-protobuf/src/proto/FunctionRpc.proto @@ -245,6 +245,14 @@ message FunctionEnvironmentReloadRequest { } message FunctionEnvironmentReloadResponse { + enum CapabilitiesUpdateStrategy { + // overwrites existing values and appends new ones + // ex. worker init: {A: foo, B: bar} + env reload: {A:foo, B: foo, C: foo} -> {A: foo, B: foo, C: foo} + merge = 0; + // existing capabilities are cleared and new capabilities are applied + // ex. worker init: {A: foo, B: bar} + env reload: {A:foo, C: foo} -> {A: foo, C: foo} + replace = 1; + } // After specialization, worker sends capabilities & metadata. // Worker metadata captured for telemetry purposes WorkerMetadata worker_metadata = 1; @@ -254,6 +262,9 @@ message FunctionEnvironmentReloadResponse { // Status of the response StatusResult result = 3; + + // If no strategy is defined, the host will default to merge + CapabilitiesUpdateStrategy capabilities_update_strategy = 4; } // Tell the out-of-proc worker to close any shared memory maps it allocated for given invocation diff --git a/release_notes.md b/release_notes.md index 37057e890..c96b86559 100644 --- a/release_notes.md +++ b/release_notes.md @@ -6,7 +6,7 @@ ### Microsoft.Azure.Functions.Worker (metapackage) -- +- Updated protobuf file to v1.10.0-protofile (#1774) ### Microsoft.Azure.Functions.Worker.Core From a1238e473c16f8941c3efdce883e94f7461d163f Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 21 Jul 2023 13:42:34 -0700 Subject: [PATCH 31/47] Add global.json to pin dotnet SDK (#1772) --- build/install-dotnet.yml | 24 ++++++++++++------------ global.json | 10 ++++++++++ 2 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 global.json diff --git a/build/install-dotnet.yml b/build/install-dotnet.yml index 255825db9..5b3e2dfc2 100644 --- a/build/install-dotnet.yml +++ b/build/install-dotnet.yml @@ -1,14 +1,14 @@ steps: -- pwsh: | - Invoke-WebRequest 'https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain/dotnet-install.ps1' -OutFile 'dotnet-install.ps1' - ./dotnet-install.ps1 -InstallDir "$env:ProgramFiles/dotnet" -Version "6.0.100" -Channel 'release' - dotnet --info - displayName: 'Install the .Net version used by the Core Tools for Windows' - condition: and(eq( variables['Agent.OS'], 'Windows_NT' ), eq(variables['FUNCTIONSRUNTIMEVERSION'], '4')) +# Some tests rely on 6.0.412 existing +- task: UseDotNet@2 + displayName: 'Install .NET6 SDK' + inputs: + packageType: 'sdk' + version: "6.0.412" -- bash: | - curl -sSL https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain/dotnet-install.sh | bash /dev/stdin -v '6.0.100' -c 'release' --install-dir /usr/share/dotnet - dotnet --info - displayName: 'Install the .Net version used by the Core Tools for Linux' - condition: and(eq( variables['Agent.OS'], 'Linux' ), eq(variables['FUNCTIONSRUNTIMEVERSION'], '4')) - +# The SDK we use to build +- task: UseDotNet@2 + displayName: 'Install current .NET SDK' + inputs: + packageType: 'sdk' + useGlobalJson: true diff --git a/global.json b/global.json new file mode 100644 index 000000000..afc32cee3 --- /dev/null +++ b/global.json @@ -0,0 +1,10 @@ +{ + "sdk": { + "version": "7.0.306", + "rollForward": "latestFeature" + }, + "msbuild-sdks": { + "Microsoft.Build.NoTargets": "3.7.0", + "Microsoft.Build.Traversal": "4.1.0" + } +} From 585b5fb22c8a795d162175d7f3cd386b01b8fbb3 Mon Sep 17 00:00:00 2001 From: JoshLove-msft <54595583+JoshLove-msft@users.noreply.github.com> Date: Tue, 13 Jun 2023 14:49:16 -0700 Subject: [PATCH 32/47] Support binding to EventData (#1609) --- .../release_notes.md | 4 +- .../src/Constants.cs | 12 ++ .../src/EventDataConverter.cs | 58 ++++++ .../src/EventHubTriggerAttribute.cs | 5 +- .../src/Properties/AssemblyInfo.cs | 2 + .../src/Worker.Extensions.EventHubs.csproj | 7 + .../EventHubs/EventDataSamples.cs | 68 ++++++ .../WorkerBindingSamples.csproj | 1 + .../FunctionMetadataGeneratorTests.cs | 71 +++++++ .../EventHubs/EventDataConverterTests.cs | 193 ++++++++++++++++++ .../Worker.Extensions.Tests.csproj | 1 + 11 files changed, 419 insertions(+), 3 deletions(-) create mode 100644 extensions/Worker.Extensions.EventHubs/src/Constants.cs create mode 100644 extensions/Worker.Extensions.EventHubs/src/EventDataConverter.cs create mode 100644 samples/WorkerBindingSamples/EventHubs/EventDataSamples.cs create mode 100644 test/Worker.Extensions.Tests/EventHubs/EventDataConverterTests.cs diff --git a/extensions/Worker.Extensions.EventHubs/release_notes.md b/extensions/Worker.Extensions.EventHubs/release_notes.md index 3a0cc9923..99426b460 100644 --- a/extensions/Worker.Extensions.EventHubs/release_notes.md +++ b/extensions/Worker.Extensions.EventHubs/release_notes.md @@ -4,6 +4,6 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Extensions.EventHubs +### Microsoft.Azure.Functions.Worker.Extensions.EventHubs 5.4.0-preview1 -- +- Add support for binding to `EventData`(#1609) diff --git a/extensions/Worker.Extensions.EventHubs/src/Constants.cs b/extensions/Worker.Extensions.EventHubs/src/Constants.cs new file mode 100644 index 000000000..070066865 --- /dev/null +++ b/extensions/Worker.Extensions.EventHubs/src/Constants.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.Functions.Worker.Extensions.EventHubs +{ + internal static class Constants + { + internal const string BinaryContentType = "application/octet-stream"; + + internal const string BindingSource = "AzureEventHubsEventData"; + } +} \ No newline at end of file diff --git a/extensions/Worker.Extensions.EventHubs/src/EventDataConverter.cs b/extensions/Worker.Extensions.EventHubs/src/EventDataConverter.cs new file mode 100644 index 000000000..71cb3e704 --- /dev/null +++ b/extensions/Worker.Extensions.EventHubs/src/EventDataConverter.cs @@ -0,0 +1,58 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Threading.Tasks; +using Azure.Core.Amqp; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using Azure.Messaging.EventHubs; +using Microsoft.Azure.Functions.Worker.Extensions.EventHubs; + +namespace Microsoft.Azure.Functions.Worker +{ + [SupportsDeferredBinding] + [SupportedConverterType(typeof(EventData))] + [SupportedConverterType(typeof(EventData[]))] + internal class EventDataConverter : IInputConverter + { + public ValueTask ConvertAsync(ConverterContext context) + { + try + { + ConversionResult result = context?.Source switch + { + ModelBindingData binding => ConversionResult.Success(ConvertToEventData(binding)), + // Only array collections are currently supported, which matches the behavior of the in-proc extension. + CollectionModelBindingData collection => ConversionResult.Success(collection.ModelBindingDataArray + .Select(ConvertToEventData).ToArray()), + _ => ConversionResult.Unhandled() + }; + return new ValueTask(result); + } + catch (Exception exception) + { + return new ValueTask(ConversionResult.Failed(exception)); + } + } + + private EventData ConvertToEventData(ModelBindingData binding) + { + if (binding?.Source is not Constants.BindingSource) + { + throw new InvalidOperationException( + $"Unexpected binding source. Only '{Constants.BindingSource}' is supported."); + } + + if (binding.ContentType != Constants.BinaryContentType) + { + throw new InvalidOperationException( + $"Unexpected content-type. Only '{Constants.BinaryContentType}' is supported."); + } + + return new EventData(AmqpAnnotatedMessage.FromBytes(binding.Content)); + } + } +} \ No newline at end of file diff --git a/extensions/Worker.Extensions.EventHubs/src/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/src/EventHubTriggerAttribute.cs index 9c8eff632..e957cdd99 100644 --- a/extensions/Worker.Extensions.EventHubs/src/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/src/EventHubTriggerAttribute.cs @@ -1,7 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; namespace Microsoft.Azure.Functions.Worker { @@ -9,6 +10,8 @@ namespace Microsoft.Azure.Functions.Worker /// Attribute used to mark a function that should be triggered by Event Hubs messages. /// [BindingCapabilities(KnownBindingCapabilities.FunctionLevelRetry)] + [AllowConverterFallback(true)] + [InputConverter(typeof(EventDataConverter))] public sealed class EventHubTriggerAttribute : TriggerBindingAttribute, ISupportCardinality { // Batch by default diff --git a/extensions/Worker.Extensions.EventHubs/src/Properties/AssemblyInfo.cs b/extensions/Worker.Extensions.EventHubs/src/Properties/AssemblyInfo.cs index fae2c1f45..32037f4c0 100644 --- a/extensions/Worker.Extensions.EventHubs/src/Properties/AssemblyInfo.cs +++ b/extensions/Worker.Extensions.EventHubs/src/Properties/AssemblyInfo.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System.Runtime.CompilerServices; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; [assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.EventHubs", "5.4.0")] +[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] diff --git a/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj b/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj index 00b49afb9..61973e733 100644 --- a/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj +++ b/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj @@ -7,13 +7,20 @@ 5.4.0 + -preview1 + + + + + + \ No newline at end of file diff --git a/samples/WorkerBindingSamples/EventHubs/EventDataSamples.cs b/samples/WorkerBindingSamples/EventHubs/EventDataSamples.cs new file mode 100644 index 000000000..f6820aaaf --- /dev/null +++ b/samples/WorkerBindingSamples/EventHubs/EventDataSamples.cs @@ -0,0 +1,68 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Azure.Messaging.EventHubs; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace SampleApp +{ + /// + /// Samples demonstrating binding to the type. + /// + public class EventDataSamples + { + private readonly ILogger _logger; + + public EventDataSamples(ILogger logger) + { + _logger = logger; + } + + /// + /// This function demonstrates binding to a single . Note that when doing so, you must also set the + /// property to false as the default value of this property is + /// true. + /// + [Function(nameof(EventDataFunctions))] + public void EventDataFunctions( + [EventHubTrigger("queue", Connection = "EventHubConnection", IsBatched = false)] EventData @event) + { + _logger.LogInformation("Event Body: {body}", @event.Body); + _logger.LogInformation("Event Content-Type: {contentType}", @event.ContentType); + } + + /// + /// This function demonstrates binding to an array of . + /// + [Function(nameof(EventDataBatchFunction))] + public void EventDataBatchFunction( + [EventHubTrigger("queue", Connection = "EventHubConnection")] EventData[] events) + { + foreach (EventData @event in events) + { + _logger.LogInformation("Event Body: {body}", @event.Body); + _logger.LogInformation("Event Content-Type: {contentType}", @event.ContentType); + } + } + + /// + /// This functions demonstrates that it is possible to bind to both the and any of the supported binding contract + /// properties at the same time. If attempting this, the must be the first parameter. There is not + /// much benefit to doing this as all of the binding contract properties are available as properties on the . + /// + [Function(nameof(EventDataWithStringPropertiesFunction))] + public void EventDataWithStringPropertiesFunction( + [EventHubTrigger("queue", Connection = "EventHubConnection")] + EventData @event, string contentType, long offset) + { + // The ContentType property and the contentType parameter are the same. + _logger.LogInformation("Event Content-Type: {contentType}", @event.ContentType); + _logger.LogInformation("Event Content-Type: {contentType}", contentType); + + // Similarly the Offset property and the offset parameter are the same. + _logger.LogInformation("Event offset: {offset}", @event.Offset); + _logger.LogInformation("Event offset: {offset}", offset); + } + } +} \ No newline at end of file diff --git a/samples/WorkerBindingSamples/WorkerBindingSamples.csproj b/samples/WorkerBindingSamples/WorkerBindingSamples.csproj index 460be44a7..85c4b51ca 100644 --- a/samples/WorkerBindingSamples/WorkerBindingSamples.csproj +++ b/samples/WorkerBindingSamples/WorkerBindingSamples.csproj @@ -14,6 +14,7 @@ + diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 3189d7dea..2d8fb7ba8 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -10,6 +10,7 @@ using System.Linq; using System.Reflection; using System.Threading.Tasks; +using Azure.Messaging.EventHubs; using Azure.Storage.Blobs; using Microsoft.Azure.Functions.Tests; using Microsoft.Azure.Functions.Worker; @@ -841,6 +842,59 @@ public void FunctionWithRetryPolicyWithInvalidIntervals() Assert.Throws(() => new ExponentialBackoffRetryAttribute(5, "something_bad", "00:01:00")); } + [Fact] + public void EventHubs_SDKTypeBindings() + { + var generator = new FunctionMetadataGenerator(); + var module = ModuleDefinition.ReadModule(_thisAssembly.Location); + var typeDef = TestUtility.GetTypeDefinition(typeof(SDKTypeBindings_EventHubs)); + var functions = generator.GenerateFunctionMetadata(typeDef); + var extensions = generator.Extensions; + + Assert.Equal(2, functions.Count()); + + AssertDictionary(extensions, new Dictionary + { + { "Microsoft.Azure.WebJobs.Extensions.EventHubs", "5.4.0" }, + }); + + var eventHubTriggerFunction = functions.Single(p => p.Name == nameof(SDKTypeBindings_EventHubs.EventHubTriggerFunction)); + + ValidateFunction(eventHubTriggerFunction, nameof(SDKTypeBindings_EventHubs.EventHubTriggerFunction), GetEntryPoint(nameof(SDKTypeBindings_EventHubs), nameof(SDKTypeBindings_EventHubs.EventHubTriggerFunction)), + ValidateEventHubTrigger); + + var eventHubBatchTriggerFunction = functions.Single(p => p.Name == nameof(SDKTypeBindings_EventHubs.EventHubBatchTriggerFunction)); + + ValidateFunction(eventHubBatchTriggerFunction, nameof(SDKTypeBindings_EventHubs.EventHubBatchTriggerFunction), GetEntryPoint(nameof(SDKTypeBindings_EventHubs), nameof(SDKTypeBindings_EventHubs.EventHubBatchTriggerFunction)), + ValidateEventHubBatchTrigger); + + void ValidateEventHubTrigger(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "event" }, + { "Type", "eventHubTrigger" }, + { "Direction", "In" }, + { "eventHubName", "hub" }, + { "Cardinality", "One" }, + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } + }); + } + + void ValidateEventHubBatchTrigger(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "events" }, + { "Type", "eventHubTrigger" }, + { "Direction", "In" }, + { "eventHubName", "hub" }, + { "Cardinality", "Many" }, + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } + }); + } + } + private class EventHubNotBatched { [Function("EventHubTrigger")] @@ -1026,6 +1080,23 @@ public object BlobStringToBlobPocoArray( } } + private class SDKTypeBindings_EventHubs + { + [Function(nameof(EventHubTriggerFunction))] + public static void EventHubTriggerFunction( + [EventHubTrigger("hub", IsBatched = false)] EventData @event) + { + throw new NotImplementedException(); + } + + [Function(nameof(EventHubBatchTriggerFunction))] + public static void EventHubBatchTriggerFunction( + [EventHubTrigger("hub")] EventData[] events) + { + throw new NotImplementedException(); + } + } + private class ExternalType_Return { public const string FunctionName = "BasicHttpWithExternalTypeReturn"; diff --git a/test/Worker.Extensions.Tests/EventHubs/EventDataConverterTests.cs b/test/Worker.Extensions.Tests/EventHubs/EventDataConverterTests.cs new file mode 100644 index 000000000..f3af27bd3 --- /dev/null +++ b/test/Worker.Extensions.Tests/EventHubs/EventDataConverterTests.cs @@ -0,0 +1,193 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Azure.Messaging.EventHubs; +using Google.Protobuf; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Extensions.EventHubs; +using Microsoft.Azure.Functions.Worker.Grpc.Messages; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.EventHubs +{ + public class EventDataConverterTests + { + [Fact] + public async Task ConvertAsync_ReturnsSuccess() + { + var eventData = CreateEventData(); + + var data = new GrpcModelBindingData(new ModelBindingData() + { + Version = "1.0", + Source = "AzureEventHubsEventData", + Content = ByteString.CopyFrom(ConvertEventDataToBinaryData(eventData)), + ContentType = Constants.BinaryContentType + }); + + var context = new TestConverterContext(typeof(string), data); + var converter = new EventDataConverter(); + var result = await converter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, result.Status); + var output = result.Value as EventData; + Assert.NotNull(output); + AssertEventData(output); + } + + [Fact] + public async Task ConvertAsync_Batch_ReturnsSuccess() + { + var message = CreateEventData(); + + var data = new ModelBindingData + { + Version = "1.0", + Source = "AzureEventHubsEventData", + Content = ByteString.CopyFrom(ConvertEventDataToBinaryData(message)), + ContentType = Constants.BinaryContentType + }; + + var array = new CollectionModelBindingData(); + array.ModelBindingData.Add(data); + array.ModelBindingData.Add(data); + + var context = new TestConverterContext(typeof(string), new GrpcCollectionModelBindingData(array)); + var converter = new EventDataConverter(); + var result = await converter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, result.Status); + var output = result.Value as EventData[]; + Assert.NotNull(output); + Assert.Equal(2, output.Length); + AssertEventData(output[0]); + AssertEventData(output[1]); + } + + [Fact] + public async Task ConvertAsync_ReturnsFailure_WrongContentType() + { + var eventData = CreateEventData(); + + var data = new GrpcModelBindingData(new ModelBindingData() + { + Version = "1.0", + Source = Constants.BindingSource, + Content = ByteString.CopyFrom(ConvertEventDataToBinaryData(eventData)), + ContentType = "application/json" + }); + + var context = new TestConverterContext(typeof(string), data); + var converter = new EventDataConverter(); + var result = await converter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, result.Status); + var output = result.Value as EventData; + Assert.Null(output); + Assert.IsType(result.Error); + } + + [Fact] + public async Task ConvertAsync_Batch_ReturnsFailure_WrongContentType() + { + var message = CreateEventData(); + + var data = new ModelBindingData + { + Version = "1.0", + Source = Constants.BindingSource, + Content = ByteString.CopyFrom(ConvertEventDataToBinaryData(message)), + ContentType = "application/json" + }; + + var array = new CollectionModelBindingData(); + array.ModelBindingData.Add(data); + array.ModelBindingData.Add(data); + + var context = new TestConverterContext(typeof(string), new GrpcCollectionModelBindingData(array)); + var converter = new EventDataConverter(); + var result = await converter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, result.Status); + var output = result.Value as EventData[]; + Assert.Null(output); + Assert.IsType(result.Error); + } + + [Fact] + public async Task ConvertAsync_ReturnsFailure_WrongSource() + { + var eventData = CreateEventData(); + + var data = new GrpcModelBindingData(new ModelBindingData() + { + Version = "1.0", + Source = "some-other-source", + Content = ByteString.CopyFrom(ConvertEventDataToBinaryData(eventData)), + ContentType = Constants.BinaryContentType + }); + + var context = new TestConverterContext(typeof(string), data); + var converter = new EventDataConverter(); + var result = await converter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, result.Status); + var output = result.Value as EventData; + Assert.Null(output); + Assert.IsType(result.Error); + } + + [Fact] + public async Task ConvertAsync_Batch_ReturnsFailure_WrongSource() + { + var message = CreateEventData(); + + var data = new ModelBindingData + { + Version = "1.0", + Source = "some-other-source", + Content = ByteString.CopyFrom(ConvertEventDataToBinaryData(message)), + ContentType = Constants.BinaryContentType + }; + + var array = new CollectionModelBindingData(); + array.ModelBindingData.Add(data); + array.ModelBindingData.Add(data); + + var context = new TestConverterContext(typeof(string), new GrpcCollectionModelBindingData(array)); + var converter = new EventDataConverter(); + var result = await converter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, result.Status); + var output = result.Value as EventData[]; + Assert.Null(output); + Assert.IsType(result.Error); + } + + private static void AssertEventData(EventData output) + { + Assert.Equal("body", output.EventBody.ToString()); + Assert.Equal("messageId", output.MessageId); + Assert.Equal("correlationId", output.CorrelationId); + Assert.Equal("contentType", output.ContentType); + } + + private static EventData CreateEventData() + { + return new EventData("body") + { + ContentType = "contentType", + CorrelationId = "correlationId", + MessageId = "messageId", + }; + } + + private static BinaryData ConvertEventDataToBinaryData(EventData @event) + { + return @event.GetRawAmqpMessage().ToBytes(); + } + } +} \ No newline at end of file diff --git a/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj b/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj index 032918cdc..6974bfd15 100644 --- a/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj +++ b/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj @@ -23,6 +23,7 @@ + From aec4408e0105aea01746d17651b6e058cf325fcf Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 14 Jun 2023 15:41:32 -0700 Subject: [PATCH 33/47] Update EventHub converter and unit tests (#1633, #1716, #1723, #1742) --------- Co-authored-by: Aishwarya Bhandari --- .../src/EventDataConverter.cs | 27 +++++--- .../src/EventHubTriggerAttribute.cs | 4 +- .../src/Worker.Extensions.EventHubs.csproj | 4 ++ .../EventHubs/EventDataSamples.cs | 2 +- .../functions.metadata | 66 +++++++++++++++++++ test/SdkE2ETests/PublishTests.cs | 3 + .../EventHubs/EventDataConverterTests.cs | 8 +-- 7 files changed, 97 insertions(+), 17 deletions(-) diff --git a/extensions/Worker.Extensions.EventHubs/src/EventDataConverter.cs b/extensions/Worker.Extensions.EventHubs/src/EventDataConverter.cs index 71cb3e704..8eeeef5ae 100644 --- a/extensions/Worker.Extensions.EventHubs/src/EventDataConverter.cs +++ b/extensions/Worker.Extensions.EventHubs/src/EventDataConverter.cs @@ -10,12 +10,16 @@ using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; using Azure.Messaging.EventHubs; using Microsoft.Azure.Functions.Worker.Extensions.EventHubs; +using Microsoft.Azure.Functions.Worker.Extensions; namespace Microsoft.Azure.Functions.Worker { + /// + /// Converter to bind to or type parameters. + /// [SupportsDeferredBinding] - [SupportedConverterType(typeof(EventData))] - [SupportedConverterType(typeof(EventData[]))] + [SupportedTargetType(typeof(EventData))] + [SupportedTargetType(typeof(EventData[]))] internal class EventDataConverter : IInputConverter { public ValueTask ConvertAsync(ConverterContext context) @@ -26,7 +30,7 @@ public ValueTask ConvertAsync(ConverterContext context) { ModelBindingData binding => ConversionResult.Success(ConvertToEventData(binding)), // Only array collections are currently supported, which matches the behavior of the in-proc extension. - CollectionModelBindingData collection => ConversionResult.Success(collection.ModelBindingDataArray + CollectionModelBindingData collection => ConversionResult.Success(collection.ModelBindingData .Select(ConvertToEventData).ToArray()), _ => ConversionResult.Unhandled() }; @@ -40,19 +44,22 @@ public ValueTask ConvertAsync(ConverterContext context) private EventData ConvertToEventData(ModelBindingData binding) { - if (binding?.Source is not Constants.BindingSource) + if (binding is null) { - throw new InvalidOperationException( - $"Unexpected binding source. Only '{Constants.BindingSource}' is supported."); + throw new ArgumentNullException(nameof(binding)); } - if (binding.ContentType != Constants.BinaryContentType) + if (binding.Source is not Constants.BindingSource) { - throw new InvalidOperationException( - $"Unexpected content-type. Only '{Constants.BinaryContentType}' is supported."); + throw new InvalidBindingSourceException(binding.Source, Constants.BindingSource); + } + + if (binding.ContentType is not Constants.BinaryContentType) + { + throw new InvalidContentTypeException(binding.ContentType, Constants.BinaryContentType); } return new EventData(AmqpAnnotatedMessage.FromBytes(binding.Content)); } } -} \ No newline at end of file +} diff --git a/extensions/Worker.Extensions.EventHubs/src/EventHubTriggerAttribute.cs b/extensions/Worker.Extensions.EventHubs/src/EventHubTriggerAttribute.cs index e957cdd99..5b768aa6a 100644 --- a/extensions/Worker.Extensions.EventHubs/src/EventHubTriggerAttribute.cs +++ b/extensions/Worker.Extensions.EventHubs/src/EventHubTriggerAttribute.cs @@ -9,9 +9,9 @@ namespace Microsoft.Azure.Functions.Worker /// /// Attribute used to mark a function that should be triggered by Event Hubs messages. /// - [BindingCapabilities(KnownBindingCapabilities.FunctionLevelRetry)] - [AllowConverterFallback(true)] [InputConverter(typeof(EventDataConverter))] + [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] + [BindingCapabilities(KnownBindingCapabilities.FunctionLevelRetry)] public sealed class EventHubTriggerAttribute : TriggerBindingAttribute, ISupportCardinality { // Batch by default diff --git a/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj b/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj index 61973e733..8607ab29c 100644 --- a/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj +++ b/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj @@ -23,4 +23,8 @@ + + + + \ No newline at end of file diff --git a/samples/WorkerBindingSamples/EventHubs/EventDataSamples.cs b/samples/WorkerBindingSamples/EventHubs/EventDataSamples.cs index f6820aaaf..5ef7192ca 100644 --- a/samples/WorkerBindingSamples/EventHubs/EventDataSamples.cs +++ b/samples/WorkerBindingSamples/EventHubs/EventDataSamples.cs @@ -53,7 +53,7 @@ public void EventDataBatchFunction( /// [Function(nameof(EventDataWithStringPropertiesFunction))] public void EventDataWithStringPropertiesFunction( - [EventHubTrigger("queue", Connection = "EventHubConnection")] + [EventHubTrigger("queue", Connection = "EventHubConnection", IsBatched = false)] EventData @event, string contentType, long offset) { // The ContentType property and the contentType parameter are the same. diff --git a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata index abc769bcc..ab0e23d58 100644 --- a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata +++ b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata @@ -386,5 +386,71 @@ "properties": {} } ] + }, + { + "name": "EventDataFunctions", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.EventDataSamples.EventDataFunctions", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "event", + "direction": "In", + "type": "eventHubTrigger", + "eventHubName": "queue", + "connection": "EventHubConnection", + "cardinality": "One", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "EventDataBatchFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.EventDataSamples.EventDataBatchFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "events", + "direction": "In", + "type": "eventHubTrigger", + "eventHubName": "queue", + "connection": "EventHubConnection", + "cardinality": "Many", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "EventDataWithStringPropertiesFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.EventDataSamples.EventDataWithStringPropertiesFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "event", + "direction": "In", + "type": "eventHubTrigger", + "eventHubName": "queue", + "connection": "EventHubConnection", + "cardinality": "One", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] } ] \ No newline at end of file diff --git a/test/SdkE2ETests/PublishTests.cs b/test/SdkE2ETests/PublishTests.cs index 6eeefa782..4022bc21e 100644 --- a/test/SdkE2ETests/PublishTests.cs +++ b/test/SdkE2ETests/PublishTests.cs @@ -122,6 +122,9 @@ private async Task RunPublishTestForSdkTypeBindings(string outputDir, string add new Extension("EventGrid", "Microsoft.Azure.WebJobs.Extensions.EventGrid.EventGridWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.EventGrid, Version=3.3.0.0, Culture=neutral, PublicKeyToken=014045d636e89289", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.EventGrid.dll"), + new Extension("EventHubs", + "Microsoft.Azure.WebJobs.EventHubs.EventHubsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.EventHubs, Version=5.4.0.0, Culture=neutral, PublicKeyToken=014045d636e89289", + @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.EventHubs.dll"), new Extension("Startup", "Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.Startup, Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader, Version=1.0.0.0, Culture=neutral, PublicKeyToken=551316b6919f366c", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.dll"), diff --git a/test/Worker.Extensions.Tests/EventHubs/EventDataConverterTests.cs b/test/Worker.Extensions.Tests/EventHubs/EventDataConverterTests.cs index f3af27bd3..3f7c90f23 100644 --- a/test/Worker.Extensions.Tests/EventHubs/EventDataConverterTests.cs +++ b/test/Worker.Extensions.Tests/EventHubs/EventDataConverterTests.cs @@ -87,7 +87,7 @@ public async Task ConvertAsync_ReturnsFailure_WrongContentType() Assert.Equal(ConversionStatus.Failed, result.Status); var output = result.Value as EventData; Assert.Null(output); - Assert.IsType(result.Error); + Assert.Equal("Unexpected content-type 'application/json'. Only 'application/octet-stream' is supported.", result.Error.Message); } [Fact] @@ -114,7 +114,7 @@ public async Task ConvertAsync_Batch_ReturnsFailure_WrongContentType() Assert.Equal(ConversionStatus.Failed, result.Status); var output = result.Value as EventData[]; Assert.Null(output); - Assert.IsType(result.Error); + Assert.Equal("Unexpected content-type 'application/json'. Only 'application/octet-stream' is supported.", result.Error.Message); } [Fact] @@ -137,7 +137,7 @@ public async Task ConvertAsync_ReturnsFailure_WrongSource() Assert.Equal(ConversionStatus.Failed, result.Status); var output = result.Value as EventData; Assert.Null(output); - Assert.IsType(result.Error); + Assert.Equal("Unexpected binding source 'some-other-source'. Only 'AzureEventHubsEventData' is supported.", result.Error.Message); } [Fact] @@ -164,7 +164,7 @@ public async Task ConvertAsync_Batch_ReturnsFailure_WrongSource() Assert.Equal(ConversionStatus.Failed, result.Status); var output = result.Value as EventData[]; Assert.Null(output); - Assert.IsType(result.Error); + Assert.Equal("Unexpected binding source 'some-other-source'. Only 'AzureEventHubsEventData' is supported.", result.Error.Message); } private static void AssertEventData(EventData output) From 8c927625c3fbe69ecb7515fe22952211eaed3d08 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Fri, 21 Jul 2023 13:57:10 -0700 Subject: [PATCH 34/47] Prepare EventHubs extension for release (#1778) --- extensions/Worker.Extensions.EventHubs/release_notes.md | 4 ++-- .../src/Worker.Extensions.EventHubs.csproj | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/extensions/Worker.Extensions.EventHubs/release_notes.md b/extensions/Worker.Extensions.EventHubs/release_notes.md index 99426b460..94990e6cf 100644 --- a/extensions/Worker.Extensions.EventHubs/release_notes.md +++ b/extensions/Worker.Extensions.EventHubs/release_notes.md @@ -4,6 +4,6 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Extensions.EventHubs 5.4.0-preview1 +### Microsoft.Azure.Functions.Worker.Extensions.EventHubs 5.5.0 -- Add support for binding to `EventData`(#1609) +- Add support for binding to `EventData` (#1609) diff --git a/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj b/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj index 8607ab29c..6c655a3cf 100644 --- a/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj +++ b/extensions/Worker.Extensions.EventHubs/src/Worker.Extensions.EventHubs.csproj @@ -6,8 +6,7 @@ Azure Event Hubs extensions for .NET isolated functions - 5.4.0 - -preview1 + 5.5.0 From 1888c9a730bb3f4da4f9a97c59f15cec3c987ac1 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Mon, 24 Jul 2023 16:49:52 -0700 Subject: [PATCH 35/47] Reset extension release notes (#1775) --- extensions/Worker.Extensions.EventGrid/release_notes.md | 4 ++-- extensions/Worker.Extensions.Storage.Blobs/release_notes.md | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/extensions/Worker.Extensions.EventGrid/release_notes.md b/extensions/Worker.Extensions.EventGrid/release_notes.md index 8113b1628..60f407532 100644 --- a/extensions/Worker.Extensions.EventGrid/release_notes.md +++ b/extensions/Worker.Extensions.EventGrid/release_notes.md @@ -4,6 +4,6 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Extensions.EventGrid 3.3.0 +### Microsoft.Azure.Functions.Worker.Extensions.EventGrid -- Add ability to bind a event grid trigger to CloudEvent, CloudEvent[], EventGridEvent, EventGridEvent[], BinaryData, BinaryData[], and string[] +- diff --git a/extensions/Worker.Extensions.Storage.Blobs/release_notes.md b/extensions/Worker.Extensions.Storage.Blobs/release_notes.md index fe008fa46..f9b50e971 100644 --- a/extensions/Worker.Extensions.Storage.Blobs/release_notes.md +++ b/extensions/Worker.Extensions.Storage.Blobs/release_notes.md @@ -4,8 +4,6 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs 6.0.0 +### Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs -- Add support for SDK-type bindings via deferred binding feature -- Remove IsBatched property from BlobInput binding attribute -- Infer binding cardinality based on blobPath +- From b9b2134b4113e088494e1f66719f3ce2ae7ed005 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Tue, 27 Jun 2023 11:14:21 -0700 Subject: [PATCH 36/47] Queue storage extension support for QueueMessage & BinaryData (#1470, #1695, #1716, #1723) --- .../release_notes.md | 2 +- .../src/Constants.cs | 14 +++ .../QueueMessageJsonConverter.cs | 91 +++++++++++++++++++ .../src/Properties/AssemblyInfo.cs | 6 +- .../src/QueueTriggerAttribute.cs | 6 +- .../src/TypeConverters/QueueConverterBase.cs | 76 ++++++++++++++++ .../QueueMessageBinaryDataConverter.cs | 45 +++++++++ .../TypeConverters/QueueMessageConverter.cs | 44 +++++++++ .../Worker.Extensions.Storage.Queues.csproj | 12 ++- .../Queue/QueueSamples.cs | 40 ++++++++ .../DotNetWorkerTests.csproj | 2 +- .../GrpcFunctionDefinitionTests.cs | 49 +++++++++- test/E2ETests/E2EApps/E2EApp/E2EApp.csproj | 2 +- .../E2EApp/Queue/QueueTestFunctions.cs | 27 +++++- test/E2ETests/E2ETests/Constants.cs | 4 + .../E2ETests/Storage/QueueEndToEndTests.cs | 36 ++++++++ .../FunctionMetadataGeneratorTests.cs | 77 +++++++++++++++- .../functions.metadata | 40 ++++++++ test/SdkE2ETests/PublishTests.cs | 8 +- .../QueueMessageBinaryDataConverterTests.cs | 84 +++++++++++++++++ .../Queue/QueueMessageConverterTests.cs | 83 +++++++++++++++++ .../Queue/QueueMessageJsonConverterTests.cs | 41 +++++++++ .../Queue/QueuesTestHelper.cs | 26 ++++++ 23 files changed, 798 insertions(+), 17 deletions(-) create mode 100644 extensions/Worker.Extensions.Storage.Queues/src/Constants.cs create mode 100644 extensions/Worker.Extensions.Storage.Queues/src/JsonConverters/QueueMessageJsonConverter.cs create mode 100644 extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueConverterBase.cs create mode 100644 extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueMessageBinaryDataConverter.cs create mode 100644 extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueMessageConverter.cs create mode 100644 samples/WorkerBindingSamples/Queue/QueueSamples.cs create mode 100644 test/Worker.Extensions.Tests/Queue/QueueMessageBinaryDataConverterTests.cs create mode 100644 test/Worker.Extensions.Tests/Queue/QueueMessageConverterTests.cs create mode 100644 test/Worker.Extensions.Tests/Queue/QueueMessageJsonConverterTests.cs create mode 100644 test/Worker.Extensions.Tests/Queue/QueuesTestHelper.cs diff --git a/extensions/Worker.Extensions.Storage.Queues/release_notes.md b/extensions/Worker.Extensions.Storage.Queues/release_notes.md index 366a8e097..76fd75a17 100644 --- a/extensions/Worker.Extensions.Storage.Queues/release_notes.md +++ b/extensions/Worker.Extensions.Storage.Queues/release_notes.md @@ -6,4 +6,4 @@ ### Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues -- +- Add ability to bind a queue trigger to QueueMessage and BinaryData (#1470) diff --git a/extensions/Worker.Extensions.Storage.Queues/src/Constants.cs b/extensions/Worker.Extensions.Storage.Queues/src/Constants.cs new file mode 100644 index 000000000..638b27003 --- /dev/null +++ b/extensions/Worker.Extensions.Storage.Queues/src/Constants.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.Functions.Worker.Storage.Queues +{ + internal static class Constants + { + internal const string QueueExtensionName = "AzureStorageQueues"; + internal const string QueueMessageText = "MessageText"; + + // Media content types + internal const string JsonContentType = "application/json"; + } +} diff --git a/extensions/Worker.Extensions.Storage.Queues/src/JsonConverters/QueueMessageJsonConverter.cs b/extensions/Worker.Extensions.Storage.Queues/src/JsonConverters/QueueMessageJsonConverter.cs new file mode 100644 index 000000000..bcd350eed --- /dev/null +++ b/extensions/Worker.Extensions.Storage.Queues/src/JsonConverters/QueueMessageJsonConverter.cs @@ -0,0 +1,91 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Azure.Storage.Queues.Models; + +namespace Microsoft.Azure.Functions.Worker.Storage.Queues +{ + internal class QueueMessageJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) => objectType == typeof(QueueMessage); + + public override QueueMessage? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType is not JsonTokenType.StartObject) + { + throw new JsonException("JSON payload expected to start with StartObject token."); + } + + string messageId = String.Empty; + string popReceipt = String.Empty; + string messageText = String.Empty; + long dequeueCount = 1; + DateTime? nextVisibleOn = null; + DateTime? insertedOn = null; + DateTime? expiresOn = null; + + var startDepth = reader.CurrentDepth; + + while (reader.Read()) + { + if (reader.TokenType is JsonTokenType.EndObject && reader.CurrentDepth == startDepth) + { + return QueuesModelFactory.QueueMessage( + messageId, + popReceipt, + messageText, + dequeueCount, + nextVisibleOn, + insertedOn, + expiresOn + ); + } + + if (reader.TokenType is not JsonTokenType.PropertyName) + { + continue; + } + + var propertyName = reader.GetString(); + reader.Read(); + + switch (propertyName?.ToLowerInvariant()) + { + case "messageid": + messageId = reader.GetString() ?? throw new JsonException("JSON payload must contain a MessageId."); + break; + case "popreceipt": + popReceipt = reader.GetString() ?? throw new JsonException("JSON payload must contain a PopReceipt."); + break; + case "messagetext": + messageText = reader.GetString() ?? throw new JsonException("JSOn payload must contain a MessageText."); + break; + case "dequeuecount": + dequeueCount = reader.GetInt64(); + break; + case "nextvisibleon": + nextVisibleOn = reader.GetDateTime(); + break; + case "insertedon": + insertedOn = reader.GetDateTime(); + break; + case "expireson": + expiresOn = reader.GetDateTime(); + break; + default: + break; + } + } + + throw new JsonException("JSON payload expected to end with EndObject token."); + } + + public override void Write(Utf8JsonWriter writer, QueueMessage value, JsonSerializerOptions options) + { + throw new JsonException($"Serialization is not supported by the {nameof(QueueMessageJsonConverter)}."); + } + } +} diff --git a/extensions/Worker.Extensions.Storage.Queues/src/Properties/AssemblyInfo.cs b/extensions/Worker.Extensions.Storage.Queues/src/Properties/AssemblyInfo.cs index 2caa76004..cd34c0800 100644 --- a/extensions/Worker.Extensions.Storage.Queues/src/Properties/AssemblyInfo.cs +++ b/extensions/Worker.Extensions.Storage.Queues/src/Properties/AssemblyInfo.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using System.Runtime.CompilerServices; +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; -[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.2")] +[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.3")] +[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] diff --git a/extensions/Worker.Extensions.Storage.Queues/src/QueueTriggerAttribute.cs b/extensions/Worker.Extensions.Storage.Queues/src/QueueTriggerAttribute.cs index 7abf908ee..56beae967 100644 --- a/extensions/Worker.Extensions.Storage.Queues/src/QueueTriggerAttribute.cs +++ b/extensions/Worker.Extensions.Storage.Queues/src/QueueTriggerAttribute.cs @@ -1,11 +1,15 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; +using System; +using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; namespace Microsoft.Azure.Functions.Worker { + [InputConverter(typeof(QueueMessageConverter))] + [InputConverter(typeof(QueueMessageBinaryDataConverter))] + [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] public sealed class QueueTriggerAttribute : TriggerBindingAttribute { private readonly string _queueName; diff --git a/extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueConverterBase.cs b/extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueConverterBase.cs new file mode 100644 index 000000000..9239196fc --- /dev/null +++ b/extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueConverterBase.cs @@ -0,0 +1,76 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Azure.Functions.Worker.Extensions; +using Microsoft.Azure.Functions.Worker.Storage.Queues; + +namespace Microsoft.Azure.Functions.Worker +{ + internal abstract class QueueConverterBase : IInputConverter + { + public QueueConverterBase() + { + } + + public bool CanConvert(ConverterContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + if (context.TargetType != typeof(T)) + { + return false; + } + + if (context.Source is not ModelBindingData bindingData) + { + return false; + } + + if (bindingData.Source is not Constants.QueueExtensionName) + { + throw new InvalidBindingSourceException(bindingData.Source, Constants.QueueExtensionName); + } + + return true; + } + + public async ValueTask ConvertAsync(ConverterContext context) + { + try + { + if (!CanConvert(context)) + { + return ConversionResult.Unhandled(); + } + + var modelBindingData = (ModelBindingData)context.Source!; + var result = await ConvertCoreAsync(modelBindingData); + return ConversionResult.Success(result); + } + catch (JsonException ex) + { + string msg = String.Format(CultureInfo.CurrentCulture, + @"Binding parameters to complex objects uses JSON serialization. + 1. Bind the parameter type as 'string' instead to get the raw values and avoid JSON deserialization, or + 2. Change the queue payload to be valid json."); + + return ConversionResult.Failed(new InvalidOperationException(msg, ex)); + } + catch (Exception ex) + { + return ConversionResult.Failed(ex); + } + } + + protected abstract ValueTask ConvertCoreAsync(ModelBindingData data); + } +} diff --git a/extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueMessageBinaryDataConverter.cs b/extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueMessageBinaryDataConverter.cs new file mode 100644 index 000000000..c35bc1b05 --- /dev/null +++ b/extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueMessageBinaryDataConverter.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Azure.Functions.Worker.Extensions; +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using Microsoft.Azure.Functions.Worker.Storage.Queues; + +namespace Microsoft.Azure.Functions.Worker +{ + /// + /// Converter to bind to type parameters. + /// + [SupportsDeferredBinding] + [SupportedTargetType(typeof(BinaryData))] + internal sealed class QueueMessageBinaryDataConverter : QueueConverterBase + { + public QueueMessageBinaryDataConverter() : base() + { + } + + protected override ValueTask ConvertCoreAsync(ModelBindingData data) + { + return new ValueTask(ExtractQueueMessageContent(data)); + } + + private BinaryData ExtractQueueMessageContent(ModelBindingData modelBindingData) + { + if (modelBindingData.ContentType is not Constants.JsonContentType) + { + throw new InvalidContentTypeException(modelBindingData.ContentType, Constants.JsonContentType); + } + + var content = modelBindingData.Content.ToObjectFromJson(); + var messageText = content.GetProperty(Constants.QueueMessageText).ToString() + ?? throw new InvalidOperationException($"The '{Constants.QueueMessageText}' property is missing or null."); + + return new BinaryData(messageText); + } + } +} diff --git a/extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueMessageConverter.cs b/extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueMessageConverter.cs new file mode 100644 index 000000000..02e319657 --- /dev/null +++ b/extensions/Worker.Extensions.Storage.Queues/src/TypeConverters/QueueMessageConverter.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Storage.Queues.Models; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Azure.Functions.Worker.Extensions; +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using Microsoft.Azure.Functions.Worker.Storage.Queues; + +namespace Microsoft.Azure.Functions.Worker +{ + /// + /// Converter to bind to type parameters. + /// + [SupportsDeferredBinding] + [SupportedTargetType(typeof(QueueMessage))] + internal sealed class QueueMessageConverter : QueueConverterBase + { + private readonly JsonSerializerOptions _jsonOptions; + + public QueueMessageConverter() : base() + { + _jsonOptions = new() { Converters = { new QueueMessageJsonConverter() } }; + } + + protected override ValueTask ConvertCoreAsync(ModelBindingData data) + { + return new ValueTask(ExtractQueueMessage(data)); + } + + private QueueMessage ExtractQueueMessage(ModelBindingData modelBindingData) + { + if (modelBindingData.ContentType is not Constants.JsonContentType) + { + throw new InvalidContentTypeException(modelBindingData.ContentType, Constants.JsonContentType); + } + + return modelBindingData.Content.ToObjectFromJson(_jsonOptions); + } + } +} diff --git a/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj b/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj index 33ca629d9..30fff20b9 100644 --- a/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj +++ b/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj @@ -6,7 +6,8 @@ Azure Queue Storage extensions for .NET isolated functions - 5.1.2 + 5.1.3 + -preview1 false @@ -16,6 +17,15 @@ + + + + + + + + + \ No newline at end of file diff --git a/samples/WorkerBindingSamples/Queue/QueueSamples.cs b/samples/WorkerBindingSamples/Queue/QueueSamples.cs new file mode 100644 index 000000000..c1240c8b3 --- /dev/null +++ b/samples/WorkerBindingSamples/Queue/QueueSamples.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Azure.Storage.Queues.Models; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace SampleApp +{ + /// + /// Samples demonstrating binding to and types. + /// + public class QueueSamples + { + private readonly ILogger _logger; + + public QueueSamples(ILogger logger) + { + _logger = logger; + } + + /// + /// This function demonstrates binding to a single . + /// + [Function(nameof(QueueMessageFunction))] + public void QueueMessageFunction([QueueTrigger("input-queue")] QueueMessage message) + { + _logger.LogInformation(message.MessageText); + } + + /// + /// This function demonstrates binding to a single . + /// + [Function(nameof(QueueBinaryDataFunction))] + public void QueueBinaryDataFunction([QueueTrigger("input-queue-binarydata")] BinaryData message) + { + _logger.LogInformation(message.ToString()); + } + } +} \ No newline at end of file diff --git a/test/DotNetWorkerTests/DotNetWorkerTests.csproj b/test/DotNetWorkerTests/DotNetWorkerTests.csproj index 6624a9c84..7331de5af 100644 --- a/test/DotNetWorkerTests/DotNetWorkerTests.csproj +++ b/test/DotNetWorkerTests/DotNetWorkerTests.csproj @@ -24,10 +24,10 @@ - + diff --git a/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs b/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs index 0c1ebca8f..8314bf07c 100644 --- a/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs +++ b/test/DotNetWorkerTests/GrpcFunctionDefinitionTests.cs @@ -4,8 +4,8 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Threading; +using Azure.Storage.Queues.Models; using Microsoft.Azure.Functions.Tests; using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Core; @@ -146,6 +146,46 @@ public void GrpcFunctionDefinition_BlobInput_Creates() }); } + [Fact] + public void GrpcFunctionDefinition_QueueTrigger_Creates() + { + using var testVariables = new TestScopedEnvironmentVariable("FUNCTIONS_WORKER_DIRECTORY", "."); + + var bindingInfoProvider = new DefaultOutputBindingsInfoProvider(); + var methodInfoLocator = new DefaultMethodInfoLocator(); + + string fullPathToThisAssembly = GetType().Assembly.Location; + var functionLoadRequest = new FunctionLoadRequest + { + FunctionId = "abc", + Metadata = new RpcFunctionMetadata + { + EntryPoint = $"Microsoft.Azure.Functions.Worker.Tests.{nameof(GrpcFunctionDefinitionTests)}+{nameof(MyQueueFunctionClass)}.{nameof(MyQueueFunctionClass.Run)}", + ScriptFile = Path.GetFileName(fullPathToThisAssembly), + Name = "myfunction" + } + }; + + FunctionDefinition definition = functionLoadRequest.ToFunctionDefinition(methodInfoLocator); + + Assert.Equal(functionLoadRequest.FunctionId, definition.Id); + Assert.Equal(functionLoadRequest.Metadata.EntryPoint, definition.EntryPoint); + Assert.Equal(functionLoadRequest.Metadata.Name, definition.Name); + Assert.Equal(fullPathToThisAssembly, definition.PathToAssembly); + + // Parameters + Assert.Collection(definition.Parameters, + q => + { + Assert.Equal("message", q.Name); + Assert.Equal(typeof(QueueMessage), q.Type); + Assert.Contains(PropertyBagKeys.ConverterFallbackBehavior, q.Properties.Keys); + Assert.Contains(PropertyBagKeys.BindingAttributeSupportedConverters, q.Properties.Keys); + Assert.Equal("Default", q.Properties[PropertyBagKeys.ConverterFallbackBehavior].ToString()); + Assert.Contains(new Dictionary>().ToString(), q.Properties[PropertyBagKeys.BindingAttributeSupportedConverters].ToString()); + }); + } + private class MyFunctionClass { public HttpResponseData Run(HttpRequestData req) @@ -172,5 +212,12 @@ public HttpResponseData Run( } } + private class MyQueueFunctionClass + { + public static void Run([QueueTrigger("input-queue")] QueueMessage message) + { + throw new NotImplementedException(); + } + } } } diff --git a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj index a170f9c3b..89aa677e5 100644 --- a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj +++ b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj @@ -42,8 +42,8 @@ - + diff --git a/test/E2ETests/E2EApps/E2EApp/Queue/QueueTestFunctions.cs b/test/E2ETests/E2EApps/E2EApp/Queue/QueueTestFunctions.cs index e59062d73..4ca6e515d 100644 --- a/test/E2ETests/E2EApps/E2EApp/Queue/QueueTestFunctions.cs +++ b/test/E2ETests/E2EApps/E2EApp/Queue/QueueTestFunctions.cs @@ -1,12 +1,11 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Linq; using System.Net; -using System.Text.Json.Serialization; -using Microsoft.AspNetCore.WebUtilities; -using Microsoft.Azure.Functions.Worker; +using Azure.Storage.Queues.Models; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.Extensions.Logging; @@ -135,6 +134,28 @@ public TestData QueueTriggerMetadata( return testData; } + [Function(nameof(QueueMessageQueueTriggerAndOutput))] + [QueueOutput("test-output-dotnet-isolated-queuemessage")] + public string QueueMessageQueueTriggerAndOutput([QueueTrigger("test-input-dotnet-isolated-queuemessage")] QueueMessage message, + FunctionContext context) + { + var logger = context.GetLogger(); + logger.LogInformation($"Message: {message}"); + + return message.Body.ToString(); + } + + [Function(nameof(BinaryDataQueueTriggerAndOutput))] + [QueueOutput("test-output-dotnet-isolated-binarydata")] + public string BinaryDataQueueTriggerAndOutput([QueueTrigger("test-input-dotnet-isolated-binarydata")] BinaryData message, + FunctionContext context) + { + var logger = context.GetLogger(); + logger.LogInformation($"Message: {message.ToString()}"); + + return message.ToString(); + } + public class TestData { public string Id { get; set; } diff --git a/test/E2ETests/E2ETests/Constants.cs b/test/E2ETests/E2ETests/Constants.cs index f9cd316f0..f1ee25351 100644 --- a/test/E2ETests/E2ETests/Constants.cs +++ b/test/E2ETests/E2ETests/Constants.cs @@ -28,6 +28,10 @@ public static class Queue public const string InputBindingNamePOCO = "test-input-dotnet-isolated-poco"; public const string InputBindingNameMetadata = "test-input-dotnet-isolated-metadata"; public const string OutputBindingNameMetadata = "test-output-dotnet-isolated-metadata"; + public const string InputBindingNameQueueMessage = "test-input-dotnet-isolated-queuemessage"; + public const string OutputBindingNameQueueMessage = "test-output-dotnet-isolated-queuemessage"; + public const string InputBindingNameBinaryData = "test-input-dotnet-isolated-binarydata"; + public const string OutputBindingNameBinaryData = "test-output-dotnet-isolated-binarydata"; public const string TestQueueMessage = "Hello, World"; } diff --git a/test/E2ETests/E2ETests/Storage/QueueEndToEndTests.cs b/test/E2ETests/E2ETests/Storage/QueueEndToEndTests.cs index ed81d8cfa..46186d1ce 100644 --- a/test/E2ETests/E2ETests/Storage/QueueEndToEndTests.cs +++ b/test/E2ETests/E2ETests/Storage/QueueEndToEndTests.cs @@ -166,5 +166,41 @@ public async Task QueueOutput_PocoList_Succeeds() IEnumerable queueMessages = await StorageHelpers.ReadMessagesFromQueue(Constants.Queue.OutputBindingNamePOCO); Assert.True(queueMessages.All(msg => msg.Contains(expectedQueueMessage))); } + + [Fact] + public async Task QueueMessageQueueTriggerAndOutput() + { + string expectedQueueMessage = Guid.NewGuid().ToString(); + + //Clear queue + await StorageHelpers.ClearQueue(Constants.Queue.OutputBindingNameQueueMessage); + await StorageHelpers.ClearQueue(Constants.Queue.InputBindingNameQueueMessage); + + //Set up and trigger + await StorageHelpers.CreateQueue(Constants.Queue.OutputBindingNameQueueMessage); + await StorageHelpers.InsertIntoQueue(Constants.Queue.InputBindingNameQueueMessage, expectedQueueMessage); + + //Verify + var queueMessage = await StorageHelpers.ReadFromQueue(Constants.Queue.OutputBindingNameQueueMessage); + Assert.Equal(expectedQueueMessage, queueMessage); + } + + [Fact] + public async Task BinaryDataQueueTriggerAndOutput() + { + string expectedQueueMessage = Guid.NewGuid().ToString(); + + //Clear queue + await StorageHelpers.ClearQueue(Constants.Queue.OutputBindingNameBinaryData); + await StorageHelpers.ClearQueue(Constants.Queue.InputBindingNameBinaryData); + + //Set up and trigger + await StorageHelpers.CreateQueue(Constants.Queue.OutputBindingNameBinaryData); + await StorageHelpers.InsertIntoQueue(Constants.Queue.InputBindingNameBinaryData, expectedQueueMessage); + + //Verify + var queueMessage = await StorageHelpers.ReadFromQueue(Constants.Queue.OutputBindingNameBinaryData); + Assert.Equal(expectedQueueMessage, queueMessage); + } } } diff --git a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs index 2d8fb7ba8..a3e97fd2f 100644 --- a/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs +++ b/test/FunctionMetadataGeneratorTests/FunctionMetadataGeneratorTests.cs @@ -12,6 +12,7 @@ using System.Threading.Tasks; using Azure.Messaging.EventHubs; using Azure.Storage.Blobs; +using Azure.Storage.Queues.Models; using Microsoft.Azure.Functions.Tests; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; @@ -172,7 +173,7 @@ public void StorageFunctions() AssertDictionary(extensions, new Dictionary { - { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.2" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.3" }, { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.3" }, }); @@ -461,6 +462,57 @@ void ValidateTrigger(ExpandoObject b) } } + [Fact] + public void QueueStorageFunctions_SDKTypeBindings() + { + var generator = new FunctionMetadataGenerator(); + var module = ModuleDefinition.ReadModule(_thisAssembly.Location); + var typeDef = TestUtility.GetTypeDefinition(typeof(SDKTypeBindings_Queue)); + var functions = generator.GenerateFunctionMetadata(typeDef); + var extensions = generator.Extensions; + + Assert.Equal(2, functions.Count()); + + AssertDictionary(extensions, new Dictionary + { + { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.3" }, + }); + + var queueMessageTriggerFunction = functions.Single(p => p.Name == nameof(SDKTypeBindings_Queue.QueueMessageTrigger)); + + ValidateFunction(queueMessageTriggerFunction, nameof(SDKTypeBindings_Queue.QueueMessageTrigger), GetEntryPoint(nameof(SDKTypeBindings_Queue), nameof(SDKTypeBindings_Queue.QueueMessageTrigger)), + ValidateQueueMessageTrigger); + + var queueBinaryDataTriggerFunction = functions.Single(p => p.Name == nameof(SDKTypeBindings_Queue.QueueBinaryDataTrigger)); + + ValidateFunction(queueBinaryDataTriggerFunction, nameof(SDKTypeBindings_Queue.QueueBinaryDataTrigger), GetEntryPoint(nameof(SDKTypeBindings_Queue), nameof(SDKTypeBindings_Queue.QueueBinaryDataTrigger)), + ValidateQueueBinaryDataTrigger); + + void ValidateQueueMessageTrigger(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "message" }, + { "Type", "queueTrigger" }, + { "Direction", "In" }, + { "queueName", "queue" }, + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } + }); + } + + void ValidateQueueBinaryDataTrigger(ExpandoObject b) + { + AssertExpandoObject(b, new Dictionary + { + { "Name", "message" }, + { "Type", "queueTrigger" }, + { "Direction", "In" }, + { "queueName", "queue" }, + { "Properties", new Dictionary( ) { { "SupportsDeferredBinding" , "True"} } } + }); + } + } + [Fact] public void MultiOutput_OnReturnType() { @@ -481,7 +533,7 @@ public void MultiOutput_OnReturnType() AssertDictionary(extensions, new Dictionary { - { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.2" }, + { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.3" }, { "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs", "5.1.3" }, }); @@ -547,7 +599,7 @@ public void MultiOutput_OnReturnType_WithHttp() AssertDictionary(extensions, new Dictionary { - { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.2" } + { "Microsoft.Azure.WebJobs.Extensions.Storage.Queues", "5.1.3" } }); void ValidateHttpTrigger(ExpandoObject b) @@ -1097,6 +1149,25 @@ public static void EventHubBatchTriggerFunction( } } + private class SDKTypeBindings_Queue + { + [Function(nameof(QueueMessageTrigger))] + public static void QueueMessageTrigger( + [QueueTrigger("queue")] QueueMessage message) + { + throw new NotImplementedException(); + } + + + [Function(nameof(QueueBinaryDataTrigger))] + public static void QueueBinaryDataTrigger( + [QueueTrigger("queue")] BinaryData message) + + { + throw new NotImplementedException(); + } + } + private class ExternalType_Return { public const string FunctionName = "BasicHttpWithExternalTypeReturn"; diff --git a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata index ab0e23d58..fe8e264a7 100644 --- a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata +++ b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata @@ -452,5 +452,45 @@ } } ] + }, + { + "name": "QueueMessageFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.QueueSamples.QueueMessageFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "message", + "direction": "In", + "type": "queueTrigger", + "queueName": "input-queue", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "QueueBinaryDataFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.QueueSamples.QueueBinaryDataFunction", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "message", + "direction": "In", + "type": "queueTrigger", + "queueName": "input-queue-binarydata", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] } ] \ No newline at end of file diff --git a/test/SdkE2ETests/PublishTests.cs b/test/SdkE2ETests/PublishTests.cs index 4022bc21e..eb81ee705 100644 --- a/test/SdkE2ETests/PublishTests.cs +++ b/test/SdkE2ETests/PublishTests.cs @@ -67,7 +67,7 @@ private async Task RunPublishTest(string outputDir, string additionalParams = nu "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageBlobsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Blobs, Version=5.1.3.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.dll"), new Extension("AzureStorageQueues", - "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageQueuesWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Queues, Version=5.1.2.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", + "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageQueuesWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Queues, Version=5.1.3.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Queues.dll") } }); @@ -77,7 +77,6 @@ private async Task RunPublishTest(string outputDir, string additionalParams = nu TestUtility.ValidateFunctionsMetadata(functionsMetadataPath, "Microsoft.Azure.Functions.SdkE2ETests.Contents.functions.metadata"); } - [Fact] public async Task Publish_SdkTypeBindings() { @@ -130,7 +129,10 @@ private async Task RunPublishTestForSdkTypeBindings(string outputDir, string add @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.FunctionMetadataLoader.dll"), new Extension("AzureStorageBlobs", "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageBlobsWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Blobs, Version=5.1.3.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", - @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.dll") + @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.dll"), + new Extension("AzureStorageQueues", + "Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageQueuesWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage.Queues, Version=5.1.3.0, Culture=neutral, PublicKeyToken=92742159e12e44c8", + @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.Storage.Queues.dll") } }); Assert.True(JToken.DeepEquals(extensionsJsonContents, expected), $"Actual: {extensionsJsonContents}{Environment.NewLine}Expected: {expected}"); diff --git a/test/Worker.Extensions.Tests/Queue/QueueMessageBinaryDataConverterTests.cs b/test/Worker.Extensions.Tests/Queue/QueueMessageBinaryDataConverterTests.cs new file mode 100644 index 000000000..c92e34f5e --- /dev/null +++ b/test/Worker.Extensions.Tests/Queue/QueueMessageBinaryDataConverterTests.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.Hosting; +using Xunit; + +// AzureStorageQueues + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Queue +{ + public class QueueMessageBinaryDataConverterTests + { + public QueueMessageBinaryDataConverterTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + } + + [Fact] + public async Task ConvertAsync_ValidModelBindingData_BinaryData_ReturnsSuccess() + { + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(QueuesTestHelper.GetTestBinaryData(), "AzureStorageQueues"); + var context = new TestConverterContext(typeof(BinaryData), grpcModelBindingData); + + var queueMessageConverter = new QueueMessageBinaryDataConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + var expectedData = conversionResult.Value as BinaryData; + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("hello world", expectedData.ToString()); + } + + [Fact] + public async Task ConvertAsync_ContentSource_AsObject_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(BinaryData), new Object()); + + var queueMessageConverter = new QueueMessageBinaryDataConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingData_Null_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(BinaryData), null); + + var queueMessageConverter = new QueueMessageBinaryDataConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingDataSource_NotQueueStorageExtension_ReturnsFailed() + { + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(QueuesTestHelper.GetTestBinaryData(), source: "anotherExtensions"); + var context = new TestConverterContext(typeof(BinaryData), grpcModelBindingData); + + var queueMessageConverter = new QueueMessageBinaryDataConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unexpected binding source 'anotherExtensions'. Only 'AzureStorageQueues' is supported.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ModelBindingDataContentType_Unsupported_ReturnsFailed() + { + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(QueuesTestHelper.GetTestBinaryData(), "AzureStorageQueues", contentType: "binary"); + var context = new TestConverterContext(typeof(BinaryData), grpcModelBindingData); + + var queueMessageConverter = new QueueMessageBinaryDataConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unexpected content-type 'binary'. Only 'application/json' is supported.", conversionResult.Error.Message); + } + } +} diff --git a/test/Worker.Extensions.Tests/Queue/QueueMessageConverterTests.cs b/test/Worker.Extensions.Tests/Queue/QueueMessageConverterTests.cs new file mode 100644 index 000000000..05c20b8b0 --- /dev/null +++ b/test/Worker.Extensions.Tests/Queue/QueueMessageConverterTests.cs @@ -0,0 +1,83 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Azure.Storage.Queues.Models; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.Hosting; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Queue +{ + public class QueueMessageConverterTests + { + public QueueMessageConverterTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + } + + [Fact] + public async Task ConvertAsync_ValidModelBindingData_QueueMessage_ReturnsSuccess() + { + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(QueuesTestHelper.GetTestBinaryData(), "AzureStorageQueues"); + var context = new TestConverterContext(typeof(QueueMessage), grpcModelBindingData); + + var queueMessageConverter = new QueueMessageConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + var expectedData = conversionResult.Value as QueueMessage; + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("hello world", expectedData.Body.ToString()); + } + + [Fact] + public async Task ConvertAsync_ContentSource_AsObject_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(QueueMessage), new Object()); + + var queueMessageConverter = new QueueMessageConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingData_Null_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(QueueMessage), null); + + var queueMessageConverter = new QueueMessageConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingDataSource_NotQueueStorageExtension_ReturnsFailed() + { + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(QueuesTestHelper.GetTestBinaryData(), source: "anotherExtensions"); + var context = new TestConverterContext(typeof(QueueMessage), grpcModelBindingData); + + var queueMessageConverter = new QueueMessageConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unexpected binding source 'anotherExtensions'. Only 'AzureStorageQueues' is supported.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ModelBindingDataContentType_Unsupported_ReturnsFailed() + { + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(QueuesTestHelper.GetTestBinaryData(), "AzureStorageQueues", contentType: "binary"); + var context = new TestConverterContext(typeof(QueueMessage), grpcModelBindingData); + + var queueMessageConverter = new QueueMessageConverter(); + var conversionResult = await queueMessageConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unexpected content-type 'binary'. Only 'application/json' is supported.", conversionResult.Error.Message); + } + } +} diff --git a/test/Worker.Extensions.Tests/Queue/QueueMessageJsonConverterTests.cs b/test/Worker.Extensions.Tests/Queue/QueueMessageJsonConverterTests.cs new file mode 100644 index 000000000..cfa1ed9f9 --- /dev/null +++ b/test/Worker.Extensions.Tests/Queue/QueueMessageJsonConverterTests.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Text.Json; +using Azure.Storage.Queues.Models; +using Microsoft.Azure.Functions.Worker.Storage.Queues; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Queue +{ + public class QueueMessageJsonConverterTests + { + [Fact] + public void QueueMessageJsonConverter_ValidJson_ReturnsQueueMessage() + { + var data = GetTestBinaryData(); + + JsonSerializerOptions options = new() { Converters = { new QueueMessageJsonConverter() } }; + var result = data.ToObjectFromJson(options); + + Assert.Equal(typeof(QueueMessage), result.GetType()); + } + + private BinaryData GetTestBinaryData(string messageId = "fbb84c41-9f1f-4c75-950c-72d0541fb8ae", string message = "hello world") + { + string jsonData = $@"{{ + ""MessageId"" : ""{messageId}"", + ""PopReceipt"" : ""AgAAAAMAAAAAAAAASm\u002B7xBZv2QE="", + ""MessageText"" : ""{message}"", + ""Body"" : {{}}, + ""NextVisibleOn"" : ""2023-04-14T21:19:16+00:00"", + ""InsertedOn"" : ""2023-04-14T21:09:14+00:00"", + ""ExpiresOn"" : ""2023-04-21T21:09:14+00:00"", + ""DequeueCount"" : 1 + }}"; + + return new BinaryData(jsonData); + } + } +} diff --git a/test/Worker.Extensions.Tests/Queue/QueuesTestHelper.cs b/test/Worker.Extensions.Tests/Queue/QueuesTestHelper.cs new file mode 100644 index 000000000..0967a15a9 --- /dev/null +++ b/test/Worker.Extensions.Tests/Queue/QueuesTestHelper.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Queue +{ + internal static class QueuesTestHelper + { + public static BinaryData GetTestBinaryData(string messageId = "fbb84c41-9f1f-4c75-950c-72d0541fb8ae", string message = "hello world") + { + string jsonData = $@"{{ + ""MessageId"" : ""{messageId}"", + ""PopReceipt"" : ""AgAAAAMAAAAAAAAASm\u002B7xBZv2QE="", + ""MessageText"" : ""{message}"", + ""Body"" : {{}}, + ""NextVisibleOn"" : ""2023-04-14T21:19:16+00:00"", + ""InsertedOn"" : ""2023-04-14T21:09:14+00:00"", + ""ExpiresOn"" : ""2023-04-21T21:09:14+00:00"", + ""DequeueCount"" : 1 + }}"; + + return new BinaryData(jsonData); + } + } +} From 383a781214e1e498b0123bec13152dc04a53aa78 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Fri, 21 Jul 2023 13:25:52 -0700 Subject: [PATCH 37/47] Prepare Queue & Storage extension pkgs for release (#1777) --- .../Worker.Extensions.Storage.Blobs/release_notes.md | 9 --------- .../release_notes.md | 9 --------- .../src/Worker.Extensions.Storage.Queues.csproj | 1 - .../Worker.Extensions.Storage/release_notes.md | 12 +++++++----- .../src/Worker.Extensions.Storage.csproj | 2 +- 5 files changed, 8 insertions(+), 25 deletions(-) delete mode 100644 extensions/Worker.Extensions.Storage.Blobs/release_notes.md delete mode 100644 extensions/Worker.Extensions.Storage.Queues/release_notes.md diff --git a/extensions/Worker.Extensions.Storage.Blobs/release_notes.md b/extensions/Worker.Extensions.Storage.Blobs/release_notes.md deleted file mode 100644 index f9b50e971..000000000 --- a/extensions/Worker.Extensions.Storage.Blobs/release_notes.md +++ /dev/null @@ -1,9 +0,0 @@ -## What's Changed - - - -### Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs - -- diff --git a/extensions/Worker.Extensions.Storage.Queues/release_notes.md b/extensions/Worker.Extensions.Storage.Queues/release_notes.md deleted file mode 100644 index 76fd75a17..000000000 --- a/extensions/Worker.Extensions.Storage.Queues/release_notes.md +++ /dev/null @@ -1,9 +0,0 @@ -## What's Changed - - - -### Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues - -- Add ability to bind a queue trigger to QueueMessage and BinaryData (#1470) diff --git a/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj b/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj index 30fff20b9..f80ba7034 100644 --- a/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj +++ b/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj @@ -7,7 +7,6 @@ 5.1.3 - -preview1 false diff --git a/extensions/Worker.Extensions.Storage/release_notes.md b/extensions/Worker.Extensions.Storage/release_notes.md index 0d51865b4..dfda11795 100644 --- a/extensions/Worker.Extensions.Storage/release_notes.md +++ b/extensions/Worker.Extensions.Storage/release_notes.md @@ -4,14 +4,16 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Extensions.Storage (metapackage) +### Microsoft.Azure.Functions.Worker.Extensions.Storage 6.0.0 - -### Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs (delete if not updated) +### Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs 6.0.0 -- +- Add support for SDK-type bindings via deferred binding feature +- Remove IsBatched property from BlobInput binding attribute +- Infer binding cardinality based on blobPath -### Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues (delete if not updated) +### Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues 5.1.3 -- +- Add ability to bind a queue trigger to QueueMessage and BinaryData (#1470) diff --git a/extensions/Worker.Extensions.Storage/src/Worker.Extensions.Storage.csproj b/extensions/Worker.Extensions.Storage/src/Worker.Extensions.Storage.csproj index a039885e0..b13c0d21e 100644 --- a/extensions/Worker.Extensions.Storage/src/Worker.Extensions.Storage.csproj +++ b/extensions/Worker.Extensions.Storage/src/Worker.Extensions.Storage.csproj @@ -6,7 +6,7 @@ Azure Storage extensions for .NET isolated functions - 5.1.2 + 6.0.0 false From 3c4eb8f5231e0d3a4d11d800bfff416064222cb5 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Tue, 25 Jul 2023 10:53:51 -0700 Subject: [PATCH 38/47] Update queue extension version (#1782) --- .../src/Worker.Extensions.Storage.Queues.csproj | 2 +- extensions/Worker.Extensions.Storage/release_notes.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj b/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj index f80ba7034..7c6932861 100644 --- a/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj +++ b/extensions/Worker.Extensions.Storage.Queues/src/Worker.Extensions.Storage.Queues.csproj @@ -6,7 +6,7 @@ Azure Queue Storage extensions for .NET isolated functions - 5.1.3 + 5.2.0 false diff --git a/extensions/Worker.Extensions.Storage/release_notes.md b/extensions/Worker.Extensions.Storage/release_notes.md index dfda11795..fdbca2029 100644 --- a/extensions/Worker.Extensions.Storage/release_notes.md +++ b/extensions/Worker.Extensions.Storage/release_notes.md @@ -14,6 +14,6 @@ - Remove IsBatched property from BlobInput binding attribute - Infer binding cardinality based on blobPath -### Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues 5.1.3 +### Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues 5.2.0 - Add ability to bind a queue trigger to QueueMessage and BinaryData (#1470) From fac754634c7bcd3cdd51354ce177c95da67f0431 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Thu, 13 Jul 2023 14:39:23 -0700 Subject: [PATCH 39/47] Cosmos DB converter for SDK-type support and samples (#1406) --- .../src/Config/CosmosDBBindingOptions.cs | 52 +++ .../src/Config/CosmosDBBindingOptionsSetup.cs | 56 +++ .../src/Constants.cs | 15 + .../src/CosmosDBConverter.cs | 202 +++++++++ .../src/CosmosDBInputAttribute.cs | 12 +- .../src/CosmosExtensionStartup.cs | 29 ++ .../src/Properties/AssemblyInfo.cs | 3 + .../src/Utilities.cs | 25 ++ .../src/Worker.Extensions.CosmosDB.csproj | 11 + .../Cosmos/CosmosInputBindingFunctions.cs | 221 ++++++++++ .../Cosmos/CosmosTriggerFunction.cs | 33 ++ .../WorkerBindingSamples/Cosmos/ToDoItem.cs | 11 + .../WorkerBindingSamples.csproj | 1 + test/SdkE2ETests/PublishTests.cs | 3 + .../Cosmos/CosmosDBConverterTests.cs | 409 ++++++++++++++++++ test/Worker.Extensions.Tests/Cosmos/Helper.cs | 25 ++ .../Cosmos/UtilitiesTests.cs | 46 ++ .../Worker.Extensions.Tests.csproj | 1 + 18 files changed, 1154 insertions(+), 1 deletion(-) create mode 100644 extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs create mode 100644 extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptionsSetup.cs create mode 100644 extensions/Worker.Extensions.CosmosDB/src/Constants.cs create mode 100644 extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs create mode 100644 extensions/Worker.Extensions.CosmosDB/src/CosmosExtensionStartup.cs create mode 100644 extensions/Worker.Extensions.CosmosDB/src/Utilities.cs create mode 100644 samples/WorkerBindingSamples/Cosmos/CosmosInputBindingFunctions.cs create mode 100644 samples/WorkerBindingSamples/Cosmos/CosmosTriggerFunction.cs create mode 100644 samples/WorkerBindingSamples/Cosmos/ToDoItem.cs create mode 100644 test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs create mode 100644 test/Worker.Extensions.Tests/Cosmos/Helper.cs create mode 100644 test/Worker.Extensions.Tests/Cosmos/UtilitiesTests.cs diff --git a/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs new file mode 100644 index 000000000..21f6446ca --- /dev/null +++ b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using Azure.Core; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Functions.Worker.Extensions.CosmosDB; + +namespace Microsoft.Azure.Functions.Worker +{ + internal class CosmosDBBindingOptions + { + public string? ConnectionString { get; set; } + + public string? AccountEndpoint { get; set; } + + public TokenCredential? Credential { get; set; } + + internal string BuildCacheKey(string connectionString, string region) => $"{connectionString}|{region}"; + internal ConcurrentDictionary ClientCache { get; } = new ConcurrentDictionary(); + + internal virtual CosmosClient GetClient(string connection, string preferredLocations = "") + { + if (string.IsNullOrEmpty(connection)) + { + throw new ArgumentNullException(nameof(connection)); + } + + string cacheKey = BuildCacheKey(connection, preferredLocations); + + CosmosClientOptions cosmosClientOptions = new () + { + ConnectionMode = ConnectionMode.Gateway + }; + + if (!string.IsNullOrEmpty(preferredLocations)) + { + cosmosClientOptions.ApplicationPreferredRegions = Utilities.ParsePreferredLocations(preferredLocations); + } + + return ClientCache.GetOrAdd(cacheKey, (c) => CreateService(cosmosClientOptions)); + } + + private CosmosClient CreateService(CosmosClientOptions cosmosClientOptions) + { + return string.IsNullOrEmpty(ConnectionString) + ? new CosmosClient(AccountEndpoint, Credential, cosmosClientOptions) // AAD auth + : new CosmosClient(ConnectionString, cosmosClientOptions); // Connection string based auth + } + } +} \ No newline at end of file diff --git a/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptionsSetup.cs b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptionsSetup.cs new file mode 100644 index 000000000..60c2a374a --- /dev/null +++ b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptionsSetup.cs @@ -0,0 +1,56 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.Functions.Worker.Extensions; +using Microsoft.Azure.Functions.Worker.Extensions.CosmosDB; +using Microsoft.Extensions.Azure; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.Functions.Worker +{ + internal class CosmosDBBindingOptionsSetup : IConfigureNamedOptions + { + private readonly IConfiguration _configuration; + private readonly AzureComponentFactory _componentFactory; + + public CosmosDBBindingOptionsSetup(IConfiguration configuration, AzureComponentFactory componentFactory) + { + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + _componentFactory = componentFactory ?? throw new ArgumentNullException(nameof(componentFactory)); + } + + public void Configure(CosmosDBBindingOptions options) + { + Configure(Options.DefaultName, options); + } + + public void Configure(string name, CosmosDBBindingOptions options) + { + IConfigurationSection connectionSection = _configuration.GetWebJobsConnectionStringSection(name); + + if (!connectionSection.Exists()) + { + throw new InvalidOperationException($"Cosmos DB connection configuration '{name}' does not exist. " + + "Make sure that it is a defined App Setting."); + } + + if (!string.IsNullOrWhiteSpace(connectionSection.Value)) + { + options.ConnectionString = connectionSection.Value; + } + else + { + options.AccountEndpoint = connectionSection[Constants.AccountEndpoint]; + if (string.IsNullOrWhiteSpace(options.AccountEndpoint)) + { + throw new InvalidOperationException($"Connection should have an '{Constants.AccountEndpoint}' property or be a " + + $"string representing a connection string."); + } + + options.Credential = _componentFactory.CreateTokenCredential(connectionSection); + } + } + } +} \ No newline at end of file diff --git a/extensions/Worker.Extensions.CosmosDB/src/Constants.cs b/extensions/Worker.Extensions.CosmosDB/src/Constants.cs new file mode 100644 index 000000000..99f3cd088 --- /dev/null +++ b/extensions/Worker.Extensions.CosmosDB/src/Constants.cs @@ -0,0 +1,15 @@ + +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.Functions.Worker.Extensions.CosmosDB +{ + internal static class Constants + { + internal const string CosmosExtensionName = "CosmosDB"; + internal const string ConfigurationSectionName = "AzureWebJobs"; + internal const string ConnectionStringsSectionName = "ConnectionStrings"; + internal const string AccountEndpoint = "accountEndpoint"; + internal const string JsonContentType = "application/json"; + } +} \ No newline at end of file diff --git a/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs new file mode 100644 index 000000000..efd8a0b84 --- /dev/null +++ b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs @@ -0,0 +1,202 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Extensions.CosmosDB; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + +namespace Microsoft.Azure.Functions.Worker +{ + /// + /// Converter to bind Cosmos DB type parameters. + /// + [SupportsDeferredBinding] + internal class CosmosDBConverter : IInputConverter + { + private readonly IOptionsSnapshot _cosmosOptions; + private readonly ILogger _logger; + + public CosmosDBConverter(IOptionsSnapshot cosmosOptions, ILogger logger) + { + _cosmosOptions = cosmosOptions ?? throw new ArgumentNullException(nameof(cosmosOptions)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask ConvertAsync(ConverterContext context) + { + return context?.Source switch + { + ModelBindingData binding => await ConvertFromBindingDataAsync(context, binding), + _ => ConversionResult.Unhandled(), + }; + } + + private async ValueTask ConvertFromBindingDataAsync(ConverterContext context, ModelBindingData modelBindingData) + { + if (!IsCosmosExtension(modelBindingData)) + { + return ConversionResult.Unhandled(); + } + + try + { + var cosmosAttribute = GetBindingDataContent(modelBindingData); + object result = await ToTargetTypeAsync(context.TargetType, cosmosAttribute); + + if (result is not null) + { + return ConversionResult.Success(result); + } + } + catch (Exception ex) + { + return ConversionResult.Failed(ex); + } + + return ConversionResult.Unhandled(); + } + + private bool IsCosmosExtension(ModelBindingData bindingData) + { + if (bindingData?.Source is not Constants.CosmosExtensionName) + { + _logger.LogTrace("Source '{source}' is not supported by {converter}", bindingData?.Source, nameof(CosmosDBConverter)); + return false; + } + + return true; + } + + private CosmosDBInputAttribute GetBindingDataContent(ModelBindingData bindingData) + { + return bindingData?.ContentType switch + { + Constants.JsonContentType => bindingData.Content.ToObjectFromJson(), + _ => throw new NotSupportedException($"Unexpected content-type. Currently only '{Constants.JsonContentType}' is supported.") + }; + } + + private async Task ToTargetTypeAsync(Type targetType, CosmosDBInputAttribute cosmosAttribute) => targetType switch + { + Type _ when targetType == typeof(CosmosClient) => CreateCosmosClient(cosmosAttribute), + Type _ when targetType == typeof(Database) => CreateCosmosClient(cosmosAttribute), + Type _ when targetType == typeof(Container) => CreateCosmosClient(cosmosAttribute), + _ => await CreateTargetObjectAsync(targetType, cosmosAttribute) + }; + + private async Task CreateTargetObjectAsync(Type targetType, CosmosDBInputAttribute cosmosAttribute) + { + MethodInfo createPOCOMethod; + + if (targetType.GenericTypeArguments.Any()) + { + targetType = targetType.GenericTypeArguments.FirstOrDefault(); + + createPOCOMethod = GetType() + .GetMethod(nameof(CreatePOCOCollectionAsync), BindingFlags.Instance | BindingFlags.NonPublic) + .MakeGenericMethod(targetType); + } + else + { + createPOCOMethod = GetType() + .GetMethod(nameof(CreatePOCOAsync), BindingFlags.Instance | BindingFlags.NonPublic) + .MakeGenericMethod(targetType); + } + + + var container = CreateCosmosClient(cosmosAttribute) as Container; + + if (container is null) + { + throw new InvalidOperationException($"Unable to create Cosmos container client for '{cosmosAttribute.ContainerName}'."); + } + + return await (Task)createPOCOMethod.Invoke(this, new object[] { container, cosmosAttribute }); + } + + private async Task CreatePOCOAsync(Container container, CosmosDBInputAttribute cosmosAttribute) + { + if (String.IsNullOrEmpty(cosmosAttribute.Id) || String.IsNullOrEmpty(cosmosAttribute.PartitionKey)) + { + throw new InvalidOperationException("The 'Id' and 'PartitionKey' properties of a CosmosDB single-item input binding cannot be null or empty."); + } + + ItemResponse item = await container.ReadItemAsync(cosmosAttribute.Id, new PartitionKey(cosmosAttribute.PartitionKey)); + + if (item is null || item?.StatusCode is not System.Net.HttpStatusCode.OK || item.Resource is null) + { + throw new InvalidOperationException($"Unable to retrieve document with ID '{cosmosAttribute.Id}' and PartitionKey '{cosmosAttribute.PartitionKey}'"); + } + + return item.Resource; + } + + private async Task CreatePOCOCollectionAsync(Container container, CosmosDBInputAttribute cosmosAttribute) + { + QueryDefinition queryDefinition = null!; + if (!String.IsNullOrEmpty(cosmosAttribute.SqlQuery)) + { + queryDefinition = new QueryDefinition(cosmosAttribute.SqlQuery); + if (cosmosAttribute.SqlQueryParameters?.Count() > 0) + { + foreach (var parameter in cosmosAttribute.SqlQueryParameters) + { + queryDefinition.WithParameter(parameter.Key, parameter.Value.ToString()); + } + } + } + + PartitionKey partitionKey = String.IsNullOrEmpty(cosmosAttribute.PartitionKey) + ? PartitionKey.None + : new PartitionKey(cosmosAttribute.PartitionKey); + + QueryRequestOptions queryRequestOptions = new() { PartitionKey = partitionKey }; + + using (var iterator = container.GetItemQueryIterator(queryDefinition: queryDefinition, requestOptions: queryRequestOptions)) + { + if (iterator is null) + { + throw new InvalidOperationException($"Unable to retrieve documents for container '{container.Id}'."); + } + + return await ExtractCosmosDocumentsAsync(iterator); + } + } + + private async Task> ExtractCosmosDocumentsAsync(FeedIterator iterator) + { + var documentList = new List(); + while (iterator.HasMoreResults) + { + FeedResponse response = await iterator.ReadNextAsync(); + documentList.AddRange(response.Resource); + } + return documentList; + } + + private T CreateCosmosClient(CosmosDBInputAttribute cosmosAttribute) + { + var cosmosDBOptions = _cosmosOptions.Get(cosmosAttribute?.Connection); + CosmosClient cosmosClient = cosmosDBOptions.GetClient(cosmosAttribute?.Connection!, cosmosAttribute?.PreferredLocations!); + + Type targetType = typeof(T); + object cosmosReference = targetType switch + { + Type _ when targetType == typeof(Database) => cosmosClient.GetDatabase(cosmosAttribute?.DatabaseName), + Type _ when targetType == typeof(Container) => cosmosClient.GetContainer(cosmosAttribute?.DatabaseName, cosmosAttribute?.ContainerName), + _ => cosmosClient + }; + + return (T)cosmosReference; + } + } +} diff --git a/extensions/Worker.Extensions.CosmosDB/src/CosmosDBInputAttribute.cs b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBInputAttribute.cs index 895ec9e9b..bc6aca573 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/CosmosDBInputAttribute.cs +++ b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBInputAttribute.cs @@ -1,10 +1,14 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System.Collections.Generic; +using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; namespace Microsoft.Azure.Functions.Worker { + [AllowConverterFallback(false)] + [InputConverter(typeof(CosmosDBConverter))] public sealed class CosmosDBInputAttribute : InputBindingAttribute { /// @@ -44,7 +48,7 @@ public CosmosDBInputAttribute(string databaseName, string containerName) /// /// Optional. - /// When specified on an output binding and is true, defines the partition key + /// When specified on an output binding and is true, defines the partition key /// path for the created container. /// When specified on an input binding, specifies the partition key value for the lookup. /// May include binding parameters. @@ -67,5 +71,11 @@ public CosmosDBInputAttribute(string databaseName, string containerName) /// PreferredLocations = "East US,South Central US,North Europe" /// public string? PreferredLocations { get; set; } + + /// + /// Optional. + /// Defines the parameters to be used with the SqlQuery + /// + public IDictionary? SqlQueryParameters { get; set; } } } diff --git a/extensions/Worker.Extensions.CosmosDB/src/CosmosExtensionStartup.cs b/extensions/Worker.Extensions.CosmosDB/src/CosmosExtensionStartup.cs new file mode 100644 index 000000000..5036e3b01 --- /dev/null +++ b/extensions/Worker.Extensions.CosmosDB/src/CosmosExtensionStartup.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Extensions.Azure; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +[assembly: WorkerExtensionStartup(typeof(CosmosExtensionStartup))] + +namespace Microsoft.Azure.Functions.Worker +{ + public class CosmosExtensionStartup : WorkerExtensionStartup + { + public override void Configure(IFunctionsWorkerApplicationBuilder applicationBuilder) + { + if (applicationBuilder == null) + { + throw new ArgumentNullException(nameof(applicationBuilder)); + } + + applicationBuilder.Services.AddAzureClientsCore(); // Adds AzureComponentFactory + applicationBuilder.Services.AddOptions(); + applicationBuilder.Services.AddSingleton, CosmosDBBindingOptionsSetup>(); + } + } +} diff --git a/extensions/Worker.Extensions.CosmosDB/src/Properties/AssemblyInfo.cs b/extensions/Worker.Extensions.CosmosDB/src/Properties/AssemblyInfo.cs index ed460448a..5d16f5c3f 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/Properties/AssemblyInfo.cs +++ b/extensions/Worker.Extensions.CosmosDB/src/Properties/AssemblyInfo.cs @@ -1,6 +1,9 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System.Runtime.CompilerServices; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; [assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.CosmosDB", "4.3.0")] +[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/extensions/Worker.Extensions.CosmosDB/src/Utilities.cs b/extensions/Worker.Extensions.CosmosDB/src/Utilities.cs new file mode 100644 index 000000000..b7794f2a8 --- /dev/null +++ b/extensions/Worker.Extensions.CosmosDB/src/Utilities.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Azure.Functions.Worker.Extensions.CosmosDB +{ + internal class Utilities + { + internal static IReadOnlyList ParsePreferredLocations(string preferredRegions) + { + if (string.IsNullOrEmpty(preferredRegions)) + { + return Enumerable.Empty().ToList(); + } + + return preferredRegions + .Split(',') + .Select((region) => region.Trim()) + .Where((region) => !string.IsNullOrEmpty(region)) + .ToList(); + } + } +} \ No newline at end of file diff --git a/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj b/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj index babd3d8ca..1f74029a8 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj +++ b/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj @@ -7,6 +7,7 @@ 4.3.0 + -preview1 false @@ -14,8 +15,18 @@ + + + + + + + + + + \ No newline at end of file diff --git a/samples/WorkerBindingSamples/Cosmos/CosmosInputBindingFunctions.cs b/samples/WorkerBindingSamples/Cosmos/CosmosInputBindingFunctions.cs new file mode 100644 index 000000000..437c08bc7 --- /dev/null +++ b/samples/WorkerBindingSamples/Cosmos/CosmosInputBindingFunctions.cs @@ -0,0 +1,221 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Net; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace SampleApp +{ + public class CosmosInputBindingFunctions + { + private readonly ILogger _logger; + + public CosmosInputBindingFunctions(ILogger logger) + { + _logger = logger; + } + + [Function(nameof(DocsByUsingCosmosClient))] + public async Task DocsByUsingCosmosClient( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput("", "", Connection = "CosmosDBConnection")] CosmosClient client) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + var iterator = client.GetContainer("ToDoItems", "Items") + .GetItemQueryIterator("SELECT * FROM c"); + + while (iterator.HasMoreResults) + { + var documents = await iterator.ReadNextAsync(); + foreach (dynamic d in documents) + { + _logger.LogInformation((string)d.description); + } + } + + return req.CreateResponse(HttpStatusCode.OK); + } + + [Function(nameof(DocsByUsingDatabaseClient))] + public async Task DocsByUsingDatabaseClient( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput("ToDoItems", "", Connection = "CosmosDBConnection")] Database database) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + var iterator = database.GetContainerQueryIterator("SELECT * FROM c"); + + while (iterator.HasMoreResults) + { + var containers = await iterator.ReadNextAsync(); + foreach (dynamic c in containers) + { + _logger.LogInformation((string)c.id); + } + } + + return req.CreateResponse(HttpStatusCode.OK); + } + + [Function(nameof(DocsByUsingContainerClient))] + public async Task DocsByUsingContainerClient( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput("ToDoItems", "Items", Connection = "CosmosDBConnection")] Container container) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + var iterator = container.GetItemQueryIterator("SELECT * FROM c"); + + while (iterator.HasMoreResults) + { + var documents = await iterator.ReadNextAsync(); + foreach (dynamic d in documents) + { + _logger.LogInformation("Found ToDo item, Description={desc}", (string)d.description); + } + } + + return req.CreateResponse(HttpStatusCode.OK); + } + + [Function(nameof(DocByIdFromQueryString))] + public HttpResponseData DocByIdFromQueryString( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput( + databaseName: "ToDoItems", + containerName: "Items", + Connection = "CosmosDBConnection", + Id = "{Query.id}", + PartitionKey = "{Query.partitionKey}")] ToDoItem toDoItem) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + if (toDoItem == null) + { + _logger.LogInformation("ToDo item not found"); + } + else + { + _logger.LogInformation("Found ToDo item, Description={desc}", toDoItem.Description); + } + + return req.CreateResponse(HttpStatusCode.OK); + } + + [Function(nameof(DocByIdFromRouteData))] + public HttpResponseData DocByIdFromRouteData( + [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "todoitems/{partitionKey}/{id}")] HttpRequestData req, + [CosmosDBInput( + databaseName: "ToDoItems", + containerName: "Items", + Connection = "CosmosDBConnection", + Id = "{id}", + PartitionKey = "{partitionKey}")] ToDoItem toDoItem) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + if (toDoItem == null) + { + _logger.LogInformation($"ToDo item not found"); + } + else + { + _logger.LogInformation($"Found ToDo item, Description={toDoItem.Description}"); + } + + return req.CreateResponse(HttpStatusCode.OK); + } + + [Function(nameof(DocByIdFromRouteDataUsingSqlQuery))] + public HttpResponseData DocByIdFromRouteDataUsingSqlQuery( + [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "todoitems2/{id}")] HttpRequestData req, + [CosmosDBInput( + databaseName: "ToDoItems", + containerName: "Items", + Connection = "CosmosDBConnection", + SqlQuery = "SELECT * FROM ToDoItems t where t.id = {id}")] + IEnumerable toDoItems) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + foreach (ToDoItem toDoItem in toDoItems) + { + _logger.LogInformation(toDoItem.Description); + } + + return req.CreateResponse(HttpStatusCode.OK); + } + + [Function(nameof(DocByIdFromQueryStringUsingSqlQuery))] + public HttpResponseData DocByIdFromQueryStringUsingSqlQuery( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput( + databaseName: "ToDoItems", + containerName: "Items", + Connection = "CosmosDBConnection", + SqlQuery = "SELECT * FROM ToDoItems t where t.id = {id}")] + IEnumerable toDoItems) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + foreach (ToDoItem toDoItem in toDoItems) + { + _logger.LogInformation(toDoItem.Description); + } + + return req.CreateResponse(HttpStatusCode.OK); + } + + [Function(nameof(DocsBySqlQuery))] + public HttpResponseData DocsBySqlQuery( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput( + databaseName: "ToDoItems", + containerName: "Items", + Connection = "CosmosDBConnection", + SqlQuery = "SELECT * FROM ToDoItems t WHERE CONTAINS(t.description, 'cat')")] IEnumerable toDoItems) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + foreach (ToDoItem toDoItem in toDoItems) + { + _logger.LogInformation(toDoItem.Description); + } + + return req.CreateResponse(HttpStatusCode.OK); + } + + [Function(nameof(DocByIdFromJSON))] + public void DocByIdFromJSON( + [QueueTrigger("todoqueueforlookup")] ToDoItemLookup toDoItemLookup, + [CosmosDBInput( + databaseName: "ToDoItems", + containerName: "Items", + Connection = "CosmosDBConnection", + Id = "{ToDoItemId}", + PartitionKey = "{ToDoItemPartitionKeyValue}")] ToDoItem toDoItem) + { + _logger.LogInformation($"C# Queue trigger function processed Id={toDoItemLookup?.ToDoItemId} Key={toDoItemLookup?.ToDoItemPartitionKeyValue}"); + + if (toDoItem == null) + { + _logger.LogInformation($"ToDo item not found"); + } + else + { + _logger.LogInformation($"Found ToDo item, Description={toDoItem.Description}"); + } + } + + public class ToDoItemLookup + { + public string? ToDoItemId { get; set; } + + public string? ToDoItemPartitionKeyValue { get; set; } + } + } +} diff --git a/samples/WorkerBindingSamples/Cosmos/CosmosTriggerFunction.cs b/samples/WorkerBindingSamples/Cosmos/CosmosTriggerFunction.cs new file mode 100644 index 000000000..f7e11f6cd --- /dev/null +++ b/samples/WorkerBindingSamples/Cosmos/CosmosTriggerFunction.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace SampleApp +{ + // We cannot use SDK-type bindings with the Cosmos trigger binding. There is no way for + // the CosmosDB SDK to let us know the ID of the document that triggered the function; + // therefore we cannot create a client that is able to pull the triggering document. + public static class CosmosTriggerFunction + { + [Function(nameof(CosmosTriggerFunction))] + public static void Run([CosmosDBTrigger( + databaseName: "ToDoItems", + containerName:"TriggerItems", + Connection = "CosmosDBConnection", + CreateLeaseContainerIfNotExists = true)] IReadOnlyList todoItems, + FunctionContext context) + { + var logger = context.GetLogger(nameof(CosmosTriggerFunction)); + + if (todoItems is not null && todoItems.Any()) + { + foreach (var doc in todoItems) + { + logger.LogInformation("ToDoItem: {desc}", doc.Description); + } + } + } + } +} diff --git a/samples/WorkerBindingSamples/Cosmos/ToDoItem.cs b/samples/WorkerBindingSamples/Cosmos/ToDoItem.cs new file mode 100644 index 000000000..86027c8f2 --- /dev/null +++ b/samples/WorkerBindingSamples/Cosmos/ToDoItem.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace SampleApp +{ + public class ToDoItem + { + public string? Id { get; set; } + public string? Description { get; set; } + } +} diff --git a/samples/WorkerBindingSamples/WorkerBindingSamples.csproj b/samples/WorkerBindingSamples/WorkerBindingSamples.csproj index 85c4b51ca..629834b04 100644 --- a/samples/WorkerBindingSamples/WorkerBindingSamples.csproj +++ b/samples/WorkerBindingSamples/WorkerBindingSamples.csproj @@ -13,6 +13,7 @@ + diff --git a/test/SdkE2ETests/PublishTests.cs b/test/SdkE2ETests/PublishTests.cs index eb81ee705..138d08e76 100644 --- a/test/SdkE2ETests/PublishTests.cs +++ b/test/SdkE2ETests/PublishTests.cs @@ -118,6 +118,9 @@ private async Task RunPublishTestForSdkTypeBindings(string outputDir, string add { extensions = new[] { + new Extension("CosmosDB", + "Microsoft.Azure.WebJobs.Extensions.CosmosDB.CosmosDBWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.CosmosDB, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35", + @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.CosmosDB.dll"), new Extension("EventGrid", "Microsoft.Azure.WebJobs.Extensions.EventGrid.EventGridWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.EventGrid, Version=3.3.0.0, Culture=neutral, PublicKeyToken=014045d636e89289", @"./.azurefunctions/Microsoft.Azure.WebJobs.Extensions.EventGrid.dll"), diff --git a/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs b/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs new file mode 100644 index 000000000..0e1a4e4e8 --- /dev/null +++ b/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs @@ -0,0 +1,409 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Functions.Worker.Converters; +using Microsoft.Azure.Functions.Worker.Tests.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Cosmos +{ + public class CosmosDBConverterTests + { + private CosmosDBConverter _cosmosDBConverter; + private Mock _mockCosmosClient; + + public CosmosDBConverterTests() + { + var host = new HostBuilder().ConfigureFunctionsWorkerDefaults((WorkerOptions options) => { }).Build(); + var logger = host.Services.GetService>(); + + _mockCosmosClient = new Mock(); + + var mockCosmosOptions = new Mock(); + mockCosmosOptions + .Setup(m => m.GetClient(It.IsAny(), It.IsAny())) + .Returns(_mockCosmosClient.Object); + + var mockCosmosOptionsSnapshot = new Mock>(); + mockCosmosOptionsSnapshot + .Setup(m => m.Get(It.IsAny())) + .Returns(mockCosmosOptions.Object); + + _cosmosDBConverter = new CosmosDBConverter(mockCosmosOptionsSnapshot.Object, logger); + } + + [Fact] + public async Task ConvertAsync_ValidModelBindingData_CosmosClient_ReturnsSuccess() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var context = new TestConverterContext(typeof(CosmosClient), grpcModelBindingData); + + _mockCosmosClient.Setup(m => m.Endpoint).Returns(new Uri("https://www.example.com")); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + var expectedClient = (CosmosClient)conversionResult.Value; + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(new Uri("https://www.example.com"), expectedClient.Endpoint); + } + + [Fact] + public async Task ConvertAsync_ValidModelBindingData_DatabaseClient_ReturnsSuccess() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var context = new TestConverterContext(typeof(Database), grpcModelBindingData); + + var _mockDatabase = new Mock(); + _mockDatabase.Setup(m => m.Id).Returns("testId"); + + _mockCosmosClient + .Setup(m => m.GetDatabase(It.IsAny())) + .Returns(_mockDatabase.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + var expectedDatabase = (Database)conversionResult.Value; + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("testId", expectedDatabase.Id); + } + + [Fact] + public async Task ConvertAsync_ValidModelBindingData_ContainerClient_ReturnsSuccess() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var context = new TestConverterContext(typeof(Container), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Id).Returns("testId"); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + var expectedContainer = (Container)conversionResult.Value; + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal("testId", expectedContainer.Id); + } + + [Fact] + public async Task ConvertAsync_ValidModelBindingData_SinglePOCO_ReturnsSuccess() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); + var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); + + var expectedToDoItem = new ToDoItem() { Id = "1", Description = "Take out the rubbish" }; + + var mockResponse = new Mock>(); + mockResponse.Setup(x => x.StatusCode).Returns(System.Net.HttpStatusCode.OK); + mockResponse.Setup(x => x.Resource).Returns(expectedToDoItem); + + var mockContainer = new Mock(); + mockContainer + .Setup(m => m.ReadItemAsync(It.IsAny(), It.IsAny(), null, CancellationToken.None)) + .ReturnsAsync(mockResponse.Object); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedToDoItem, conversionResult.Value); + } + + [Fact] + public async Task ConvertAsync_ValidModelBindingData_SinglePOCO_WithoutId_ReturnsFailed() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(partitionKey: "1"), "CosmosDB"); + var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); + + var mockContainer = new Mock(); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("The 'Id' and 'PartitionKey' properties of a CosmosDB single-item input binding cannot be null or empty.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ValidModelBindingData_SinglePOCO_WithoutPK_ReturnsFailed() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1"), "CosmosDB"); + var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); + + var mockContainer = new Mock(); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("The 'Id' and 'PartitionKey' properties of a CosmosDB single-item input binding cannot be null or empty.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ValidModelBindingData_IEnumerablePOCO_ReturnsSuccess() + { + var query = "SELECT * FROM TodoItems t WHERE t.id = @id"; + var queryParams = @"{""@id"":""1""}"; + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(query: query, queryParams: queryParams), "CosmosDB"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var todo1 = new ToDoItem() { Id = "1", Description = "Take out the rubbish" }; + var todo2 = new ToDoItem() { Id = "2", Description = "Write unit tests for cosmos converter" }; + var expectedList = new List() { todo1, todo2 }; + + var mockFeedResponse = new Mock>(); + mockFeedResponse.Setup(x => x.StatusCode).Returns(System.Net.HttpStatusCode.OK); + mockFeedResponse.Setup(x => x.Resource).Returns(expectedList); + + var mockFeedIterator = new Mock>(); + mockFeedIterator.SetupSequence(x => x.HasMoreResults).Returns(true).Returns(false); + mockFeedIterator.Setup(x => x.ReadNextAsync(default)).ReturnsAsync(mockFeedResponse.Object); + + var mockContainer = new Mock(); + mockContainer + .Setup(m => m.GetItemQueryIterator(It.IsAny(), default, It.IsAny())) + .Returns(mockFeedIterator.Object); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Succeeded, conversionResult.Status); + Assert.Equal(expectedList, conversionResult.Value); + } + + [Fact] + public async Task ConvertAsync_Container_NullFeedIterator_ReturnsFailed() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Id).Returns("testId"); + mockContainer + .Setup(m => m.GetItemQueryIterator(It.IsAny(), null, null)) + .Returns>(null); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Contains("Unable to retrieve documents for container 'testId'.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ContainerWithSqlQuery_NullFeedIterator_ReturnsFailed() + { + var query = "SELECT * FROM TodoItems t WHERE t.id = @id"; + var queryParams = @"{""@id"":""1""}"; + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(query: query, queryParams: queryParams), "CosmosDB"); + var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer.Setup(m => m.Id).Returns("testId"); + mockContainer + .Setup(m => m.GetItemQueryIterator(It.IsAny(), null, null)) + .Returns((FeedIterator)null); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Contains($"Unable to retrieve documents for container 'testId'.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ContentSource_AsObject_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(CosmosClient), new Object()); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingData_Null_ReturnsUnhandled() + { + var context = new TestConverterContext(typeof(CosmosClient), null); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Unhandled, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ThrowsException_ReturnsFailure() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var context = new TestConverterContext(typeof(Database), grpcModelBindingData); + + _mockCosmosClient + .Setup(m => m.GetDatabase(It.IsAny())) + .Throws(new Exception()); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ItemResponse_ResourceIsNull_ThrowsException_ReturnsFailed() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); + var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); + + var mockResponse = new Mock>(); + mockResponse.Setup(x => x.StatusCode).Returns(System.Net.HttpStatusCode.OK); + mockResponse.Setup(x => x.Resource).Returns((ToDoItem)null); + + var mockContainer = new Mock(); + mockContainer + .Setup(m => m.ReadItemAsync(It.IsAny(), It.IsAny(), null, CancellationToken.None)) + .ReturnsAsync(mockResponse.Object); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unable to retrieve document with ID '1' and PartitionKey '1'", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_ModelBindingDataSource_NotCosmosExtension_ReturnsFailed() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "anotherExtensions"); + var context = new TestConverterContext(typeof(CosmosClient), grpcModelBindingData); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + } + + [Fact] + public async Task ConvertAsync_ModelBindingDataContentType_Unsupported_ReturnsFailed() + { + var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB", contentType: "binary"); + var context = new TestConverterContext(typeof(CosmosClient), grpcModelBindingData); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unexpected content-type. Only 'application/json' is supported.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_CosmosContainerIsNull_ThrowsException_ReturnsFailure() + { + object grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(container: "myContainer"), "CosmosDB"); + var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(null); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal($"Unable to create Cosmos container client for 'myContainer'.", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_POCO_ItemResponseNull_ThrowsException_ReturnsFailure() + { + object grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); + var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); + + var mockContainer = new Mock(); + mockContainer + .Setup(m => m.ReadItemAsync(It.IsAny(), It.IsAny(), null, CancellationToken.None)) + .Returns(Task.FromResult>(null)); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unable to retrieve document with ID '1' and PartitionKey '1'", conversionResult.Error.Message); + } + + [Fact] + public async Task ConvertAsync_POCO_IdProvided_StatusNot200_ThrowsException_ReturnsFailure() + { + object grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); + var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); + + var mockResponse = new Mock>(); + mockResponse.Setup(x => x.StatusCode).Returns(System.Net.HttpStatusCode.InternalServerError); + + var mockContainer = new Mock(); + mockContainer + .Setup(m => m.ReadItemAsync(It.IsAny(), It.IsAny(), null, CancellationToken.None)) + .ReturnsAsync(mockResponse.Object); + + _mockCosmosClient + .Setup(m => m.GetContainer(It.IsAny(), It.IsAny())) + .Returns(mockContainer.Object); + + var conversionResult = await _cosmosDBConverter.ConvertAsync(context); + + Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unable to retrieve document with ID '1' and PartitionKey '1'", conversionResult.Error.Message); + } + + private BinaryData GetTestBinaryData(string db = "testDb", string container = "testContainer", string connection = "cosmosConnection", string id = "", string partitionKey = "", string query = "", string location = "", string queryParams = "{}") + { + string jsonData = $@"{{ + ""DatabaseName"" : ""{db}"", + ""ContainerName"" : ""{container}"", + ""Connection"" : ""{connection}"", + ""Id"" : ""{id}"", + ""PartitionKey"" : ""{partitionKey}"", + ""SqlQuery"" : ""{query}"", + ""PreferredLocations"" : ""{location}"", + ""SqlQueryParameters"" : {queryParams} + }}"; + + return new BinaryData(jsonData); + } + public class ToDoItem + { + public string Id { get; set; } + public string Description { get; set; } + } + } +} \ No newline at end of file diff --git a/test/Worker.Extensions.Tests/Cosmos/Helper.cs b/test/Worker.Extensions.Tests/Cosmos/Helper.cs new file mode 100644 index 000000000..479f793e3 --- /dev/null +++ b/test/Worker.Extensions.Tests/Cosmos/Helper.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Google.Protobuf; +using Microsoft.Azure.Functions.Worker.Grpc.Messages; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Cosmos +{ + public static class Helper + { + internal static GrpcModelBindingData GetTestGrpcModelBindingData(BinaryData content, string source, string contentType = "application/json") + { + var data = new ModelBindingData() + { + Version = "1.0", + Source = source, + Content = ByteString.CopyFrom(content), + ContentType = contentType + }; + + return new GrpcModelBindingData(data); + } + } +} \ No newline at end of file diff --git a/test/Worker.Extensions.Tests/Cosmos/UtilitiesTests.cs b/test/Worker.Extensions.Tests/Cosmos/UtilitiesTests.cs new file mode 100644 index 000000000..ae01bceee --- /dev/null +++ b/test/Worker.Extensions.Tests/Cosmos/UtilitiesTests.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Azure.Functions.Worker.Extensions.CosmosDB; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Cosmos +{ + public class UtilitiesTests + { + [Theory] + [InlineData("westeurope,eastus")] + [InlineData("westeurope , eastus")] + [InlineData(" westeurope, eastus ")] + public void ParsePreferredLocations_ValidInput_ReturnsList(string input) + { + // Arrange + var expectedList = new List(); + expectedList.Add("westeurope"); + expectedList.Add("eastus"); + + // Act + var result = Utilities.ParsePreferredLocations(input); + + //Assert + Assert.Equal(expectedList, result); + } + + [Theory] + [InlineData("")] + [InlineData(null)] + public void ParsePreferredLocations_InvalidInput_ReturnsEmptyList(string input) + { + // Arrange + var expectedList = Enumerable.Empty().ToList(); + + // Act + var result = Utilities.ParsePreferredLocations(input); + + //Assert + Assert.Equal(expectedList, result); + } + } +} \ No newline at end of file diff --git a/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj b/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj index 6974bfd15..f9a70e48a 100644 --- a/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj +++ b/test/Worker.Extensions.Tests/Worker.Extensions.Tests.csproj @@ -22,6 +22,7 @@ + From 3c59cc66c1993b03bff10612f07d502aa2ae024b Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 31 May 2023 12:52:53 -0700 Subject: [PATCH 40/47] Add E2E tests for Cosmos extension (#1426) --- .../src/CosmosDBConverter.cs | 11 +- .../E2EApps/E2EApp/Cosmos/CosmosFunction.cs | 142 ++++++++++++++++-- test/E2ETests/E2EApps/E2EApp/E2EApp.csproj | 4 +- .../E2ETests/Cosmos/CosmosDBEndToEndTests.cs | 115 +++++++++++++- .../E2ETests/Fixtures/FunctionAppFixture.cs | 7 + .../E2ETests/Helpers/CosmosDBHelpers.cs | 16 +- test/E2ETests/E2ETests/Helpers/HttpHelpers.cs | 2 +- test/E2ETests/E2ETests/HttpEndToEndTests.cs | 10 +- 8 files changed, 274 insertions(+), 33 deletions(-) diff --git a/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs index efd8a0b84..76615a115 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs +++ b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs @@ -155,11 +155,12 @@ private async Task CreatePOCOCollectionAsync(Container container, Cos } } - PartitionKey partitionKey = String.IsNullOrEmpty(cosmosAttribute.PartitionKey) - ? PartitionKey.None - : new PartitionKey(cosmosAttribute.PartitionKey); - - QueryRequestOptions queryRequestOptions = new() { PartitionKey = partitionKey }; + QueryRequestOptions queryRequestOptions = new(); + if (!String.IsNullOrEmpty(cosmosAttribute.PartitionKey)) + { + var partitionKey = new PartitionKey(cosmosAttribute.PartitionKey); + queryRequestOptions = new() { PartitionKey = partitionKey }; + } using (var iterator = container.GetItemQueryIterator(queryDefinition: queryDefinition, requestOptions: queryRequestOptions)) { diff --git a/test/E2ETests/E2EApps/E2EApp/Cosmos/CosmosFunction.cs b/test/E2ETests/E2EApps/E2EApp/Cosmos/CosmosFunction.cs index 888636bd4..22bcd3521 100644 --- a/test/E2ETests/E2EApps/E2EApp/Cosmos/CosmosFunction.cs +++ b/test/E2ETests/E2EApps/E2EApp/Cosmos/CosmosFunction.cs @@ -3,20 +3,30 @@ using System.Collections.Generic; using System.Linq; -using System.Text.Json.Serialization; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Functions.Worker.Http; using Microsoft.Extensions.Logging; namespace Microsoft.Azure.Functions.Worker.E2EApp { - public static class CosmosFunction + public class CosmosFunction { + private readonly ILogger _logger; + + public CosmosFunction(ILogger logger) + { + _logger = logger; + } + [Function(nameof(CosmosTrigger))] [CosmosDBOutput( databaseName: "%CosmosDb%", containerName: "%CosmosCollOut%", Connection = "CosmosConnection", CreateIfNotExists = true)] - public static object CosmosTrigger([CosmosDBTrigger( + public object CosmosTrigger([CosmosDBTrigger( databaseName: "%CosmosDb%", containerName: "%CosmosCollIn%", Connection = "CosmosConnection", @@ -27,24 +37,136 @@ public static object CosmosTrigger([CosmosDBTrigger( { foreach (var doc in input) { - context.GetLogger("Function.CosmosTrigger").LogInformation($"id: {doc.Id}"); + context.GetLogger("Function.CosmosTrigger").LogInformation($"id: {doc.Text}"); } - return input.Select(p => new { id = p.Id }); + return input.Select(p => new { id = p.Text }); } return null; } - public class MyDocument + [Function(nameof(DocsByUsingCosmosClient))] + public async Task DocsByUsingCosmosClient( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput("", "", Connection = "CosmosConnection")] CosmosClient client) { - public string Id { get; set; } + var container = client.GetContainer("ItemDb", "ItemCollectionIn"); + var iterator = container.GetItemQueryIterator("SELECT * FROM c"); - public string Text { get; set; } + var output = ""; + + while (iterator.HasMoreResults) + { + var documents = await iterator.ReadNextAsync(); + foreach (dynamic d in documents) + { + output += $"{(string)d.Text}, "; + } + } + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(output); + return response; + } + + [Function(nameof(DocsByUsingDatabaseClient))] + public async Task DocsByUsingDatabaseClient( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput("%CosmosDb%", "", Connection = "CosmosConnection")] Database database) + { + var container = database.GetContainer("ItemCollectionIn");; + var iterator = container.GetItemQueryIterator("SELECT * FROM c"); + + var output = ""; + + while (iterator.HasMoreResults) + { + var documents = await iterator.ReadNextAsync(); + foreach (dynamic d in documents) + { + output += $"{(string)d.Text}, "; + } + } + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(output); + return response; + } + + [Function(nameof(DocsByUsingContainerClient))] + public async Task DocsByUsingContainerClient( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput("%CosmosDb%", "%CosmosCollIn%", Connection = "CosmosConnection")] Container container) + { + var iterator = container.GetItemQueryIterator("SELECT * FROM c"); + + var output = ""; + + while (iterator.HasMoreResults) + { + var documents = await iterator.ReadNextAsync(); + foreach (dynamic d in documents) + { + output += $"{(string)d.Text}, "; + } + } + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(output); + return response; + } - public int Number { get; set; } + [Function(nameof(DocByIdFromRouteData))] + public async Task DocByIdFromRouteData( + [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "docsbyroute/{partitionKey}/{id}")] HttpRequestData req, + [CosmosDBInput( + databaseName: "%CosmosDb%", + containerName: "%CosmosCollIn%", + Connection = "CosmosConnection", + Id = "{id}", + PartitionKey = "{partitionKey}")] MyDocument doc) + { + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(doc.Text); + return response; + } + + [Function(nameof(DocByIdFromRouteDataUsingSqlQuery))] + public async Task DocByIdFromRouteDataUsingSqlQuery( + [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "docsbysql/{id}")] HttpRequestData req, + [CosmosDBInput( + databaseName: "%CosmosDb%", + containerName: "%CosmosCollIn%", + Connection = "CosmosConnection", + SqlQuery = "SELECT * FROM ItemDb t where t.id = {id}")] + IEnumerable myDocs) + { + var output = myDocs.FirstOrDefault().Text; + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(output); + return response; + } - public bool Boolean { get; set; } + [Function(nameof(DocByIdFromQueryStringUsingSqlQuery))] + public async Task DocByIdFromQueryStringUsingSqlQuery( + [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, + [CosmosDBInput( + databaseName: "%CosmosDb%", + containerName: "%CosmosCollIn%", + Connection = "CosmosConnection", + SqlQuery = "SELECT * FROM ItemDb t where t.id = {id}")] + IEnumerable myDocs) + { + var output = myDocs.FirstOrDefault().Text; + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteStringAsync(output); + return response; + } + + public class MyDocument + { + public string Text { get; set; } } } } diff --git a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj index 89aa677e5..2abc7cfb3 100644 --- a/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj +++ b/test/E2ETests/E2EApps/E2EApp/E2EApp.csproj @@ -3,12 +3,12 @@ net6.0 v4 - + net7.0 v4 - + Exe <_FunctionsSkipCleanOutput>true diff --git a/test/E2ETests/E2ETests/Cosmos/CosmosDBEndToEndTests.cs b/test/E2ETests/E2ETests/Cosmos/CosmosDBEndToEndTests.cs index 71fb75934..5e1593f3c 100644 --- a/test/E2ETests/E2ETests/Cosmos/CosmosDBEndToEndTests.cs +++ b/test/E2ETests/E2ETests/Cosmos/CosmosDBEndToEndTests.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Net; +using System.Net.Http; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -27,7 +29,7 @@ public async Task CosmosDBTriggerAndOutput_Succeeds() try { //Trigger - await CosmosDBHelpers.CreateDocument(expectedDocId); + await CosmosDBHelpers.CreateDocument(expectedDocId, expectedDocId); //Read var documentId = await CosmosDBHelpers.ReadDocument(expectedDocId); @@ -40,6 +42,117 @@ public async Task CosmosDBTriggerAndOutput_Succeeds() } } + [Theory] + [InlineData("DocsByUsingCosmosClient")] + [InlineData("DocsByUsingDatabaseClient")] + [InlineData("DocsByUsingContainerClient")] + public async Task CosmosInput_ClientBinding_Succeeds(string functionName) + { + string expectedDocId = Guid.NewGuid().ToString(); + try + { + //Setup + await CosmosDBHelpers.CreateDocument(expectedDocId, expectedDocId); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionName); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Contains(expectedDocId, actualMessage); + } + finally + { + //Clean up + await CosmosDBHelpers.DeleteTestDocuments(expectedDocId); + } + } + + [Fact] + public async Task CosmosInput_DocByIdFromRouteData_Succeeds() + { + string expectedDocId = Guid.NewGuid().ToString(); + string functionPath = $"docsbyroute/{expectedDocId}/{expectedDocId}"; + try + { + //Setup + await CosmosDBHelpers.CreateDocument(expectedDocId, "DocByIdFromRouteData"); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionPath); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Contains("DocByIdFromRouteData", actualMessage); + } + finally + { + //Clean up + await CosmosDBHelpers.DeleteTestDocuments(expectedDocId); + } + } + + [Fact] + public async Task CosmosInput_DocByIdFromRouteDataUsingSqlQuery_Succeeds() + { + string expectedDocId = Guid.NewGuid().ToString(); + string functionPath = $"docsbysql/{expectedDocId}"; + try + { + //Setup + await CosmosDBHelpers.CreateDocument(expectedDocId, "DocByIdFromRouteDataUsingSqlQuery"); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionPath); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Contains("DocByIdFromRouteDataUsingSqlQuery", actualMessage); + } + finally + { + //Clean up + await CosmosDBHelpers.DeleteTestDocuments(expectedDocId); + } + } + + [Fact] + public async Task CosmosInput_DocByIdFromQueryStringUsingSqlQuery_Succeeds() + { + string expectedDocId = Guid.NewGuid().ToString(); + string functionName = "DocByIdFromQueryStringUsingSqlQuery"; + string requestBody = @$"{{ ""id"": ""{expectedDocId}"" }}"; + try + { + //Setup + await CosmosDBHelpers.CreateDocument(expectedDocId, functionName); + + //Trigger + HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody(functionName, requestBody, "application/json"); + string actualMessage = await response.Content.ReadAsStringAsync(); + + //Verify + HttpStatusCode expectedStatusCode = HttpStatusCode.OK; + + Assert.Equal(expectedStatusCode, response.StatusCode); + Assert.Contains(functionName, actualMessage); + } + finally + { + //Clean up + await CosmosDBHelpers.DeleteTestDocuments(expectedDocId); + } + } + public void Dispose() { _disposeLog?.Dispose(); diff --git a/test/E2ETests/E2ETests/Fixtures/FunctionAppFixture.cs b/test/E2ETests/E2ETests/Fixtures/FunctionAppFixture.cs index d22a3d2a7..1e76dd131 100644 --- a/test/E2ETests/E2ETests/Fixtures/FunctionAppFixture.cs +++ b/test/E2ETests/E2ETests/Fixtures/FunctionAppFixture.cs @@ -95,6 +95,13 @@ await TestUtility.RetryAsync(async () => } catch { + if (_funcProcess.HasExited) + { + // Something went wrong starting the host - check the logs + _logger.LogInformation($" Current state: process exited - something may have gone wrong."); + return false; + } + // Can get exceptions before host is running. _logger.LogInformation($" Current state: process starting"); return false; diff --git a/test/E2ETests/E2ETests/Helpers/CosmosDBHelpers.cs b/test/E2ETests/E2ETests/Helpers/CosmosDBHelpers.cs index 92d512d23..f146c3396 100644 --- a/test/E2ETests/E2ETests/Helpers/CosmosDBHelpers.cs +++ b/test/E2ETests/E2ETests/Helpers/CosmosDBHelpers.cs @@ -28,12 +28,10 @@ static CosmosDBHelpers() } // keep - public async static Task CreateDocument(string docId) + public async static Task CreateDocument(string docId, string docText = "test") { - Document documentToTest = new Document() - { - Id = docId - }; + Document documentToTest = new Document() { Id = docId }; + documentToTest.SetPropertyValue("Text", docText); _ = await _docDbClient.CreateDocumentAsync(UriFactory.CreateDocumentCollectionUri(Constants.CosmosDB.DbName, Constants.CosmosDB.InputCollectionName), documentToTest); } @@ -63,16 +61,16 @@ await TestUtility.RetryAsync(async () => public async static Task DeleteTestDocuments(string docId) { var inputDocUri = UriFactory.CreateDocumentUri(Constants.CosmosDB.DbName, Constants.CosmosDB.InputCollectionName, docId); - await DeleteDocument(inputDocUri); + await DeleteDocument(inputDocUri, docId); var outputDocUri = UriFactory.CreateDocumentUri(Constants.CosmosDB.DbName, Constants.CosmosDB.OutputCollectionName, docId); - await DeleteDocument(outputDocUri); + await DeleteDocument(outputDocUri, docId); } - private async static Task DeleteDocument(Uri docUri) + private async static Task DeleteDocument(Uri docUri, string docId) { try { - await _docDbClient.DeleteDocumentAsync(docUri); + await _docDbClient.DeleteDocumentAsync(docUri, new RequestOptions { PartitionKey = new PartitionKey(docId) }); } catch (Exception) { diff --git a/test/E2ETests/E2ETests/Helpers/HttpHelpers.cs b/test/E2ETests/E2ETests/Helpers/HttpHelpers.cs index a7fd68ed6..d5f788498 100644 --- a/test/E2ETests/E2ETests/Helpers/HttpHelpers.cs +++ b/test/E2ETests/E2ETests/Helpers/HttpHelpers.cs @@ -19,7 +19,7 @@ public static async Task InvokeHttpTrigger(string functionN return await GetResponseMessage(request); } - public static async Task InvokeHttpTriggerWithBody(string functionName, string body, HttpStatusCode expectedStatusCode, string mediaType, int expectedCode = 0) + public static async Task InvokeHttpTriggerWithBody(string functionName, string body, string mediaType) { HttpRequestMessage request = GetTestRequest(functionName); request.Content = new StringContent(body); diff --git a/test/E2ETests/E2ETests/HttpEndToEndTests.cs b/test/E2ETests/E2ETests/HttpEndToEndTests.cs index 92355ba68..828e8fd05 100644 --- a/test/E2ETests/E2ETests/HttpEndToEndTests.cs +++ b/test/E2ETests/E2ETests/HttpEndToEndTests.cs @@ -49,7 +49,7 @@ public async Task HttpTriggerTests(string functionName, string queryString, Http [InlineData("HelloFromJsonBody", "{\"Name\": \"Bob\"}", "application/octet-stream", HttpStatusCode.OK, "Hello Bob")] public async Task HttpTriggerTestsMediaTypeDoNotMatter(string functionName, string body, string mediaType, HttpStatusCode expectedStatusCode, string expectedBody) { - HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody(functionName, body, expectedStatusCode, mediaType); + HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody(functionName, body, mediaType); string responseBody = await response.Content.ReadAsStringAsync(); Assert.Equal(expectedStatusCode, response.StatusCode); @@ -62,7 +62,7 @@ public async Task HttpTriggerTestsMediaTypeDoNotMatter(string functionName, stri [InlineData("PocoAfterRouteParameters", "eu/caller/", "{ \"Name\": \"c\" }", "application/json", HttpStatusCode.OK, "eu caller c")] public async Task HttpTriggerTests_PocoFromBody(string functionName, string route, string body, string mediaType, HttpStatusCode expectedStatusCode, string expectedBody) { - HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody($"{route}{functionName}", body, expectedStatusCode, mediaType); + HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody($"{route}{functionName}", body, mediaType); string responseBody = await response.Content.ReadAsStringAsync(); Assert.Equal(expectedStatusCode, response.StatusCode); @@ -74,7 +74,7 @@ public async Task HttpTriggerTests_PocoWithoutBindingSource() { const HttpStatusCode expectedStatusCode = HttpStatusCode.InternalServerError; - HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody("PocoWithoutBindingSource", "{ \"Name\": \"John\" }", expectedStatusCode, "application/json"); + HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody("PocoWithoutBindingSource", "{ \"Name\": \"John\" }", "application/json"); Assert.Equal(expectedStatusCode, response.StatusCode); } @@ -82,7 +82,7 @@ public async Task HttpTriggerTests_PocoWithoutBindingSource() [Fact] public async Task HttpTriggerTestsPocoResult() { - HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody("HelloUsingPoco", string.Empty, HttpStatusCode.OK, "application/json"); + HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody("HelloUsingPoco", string.Empty, "application/json"); string responseBody = await response.Content.ReadAsStringAsync(); Assert.Equal(HttpStatusCode.OK, response.StatusCode); @@ -92,7 +92,7 @@ public async Task HttpTriggerTestsPocoResult() [Fact(Skip = "Proxies not currently supported in V4 but will be coming back.")] public async Task HttpProxy() { - HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody("proxytest", string.Empty, HttpStatusCode.OK, "application/json"); + HttpResponseMessage response = await HttpHelpers.InvokeHttpTriggerWithBody("proxytest", string.Empty, "application/json"); string responseBody = await response.Content.ReadAsStringAsync(); Assert.Equal(HttpStatusCode.OK, response.StatusCode); From 6194fa585e47ba11d90e407833f2c44926fe5769 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Tue, 13 Jun 2023 14:08:28 -0700 Subject: [PATCH 41/47] Refactor CosmosDB converter, samples, and attributes (#1606, #1716, #1695) --- .../release_notes.md | 4 +- .../src/Config/CosmosDBBindingOptions.cs | 13 +- .../src/Config/CosmosDBBindingOptionsSetup.cs | 8 +- .../src/CosmosDBConverter.cs | 49 ++- .../src/CosmosDBInputAttribute.cs | 26 +- .../src/Worker.Extensions.CosmosDB.csproj | 2 +- .../Cosmos/CosmosInputBindingFunctions.cs | 54 ++- .../Cosmos/CosmosTriggerFunction.cs | 25 +- .../WorkerBindingSamples/local.settings.json | 4 +- .../functions.metadata | 363 ++++++++++++++++++ .../Cosmos/CosmosDBConverterTests.cs | 37 +- test/Worker.Extensions.Tests/Cosmos/Helper.cs | 25 -- 12 files changed, 518 insertions(+), 92 deletions(-) delete mode 100644 test/Worker.Extensions.Tests/Cosmos/Helper.cs diff --git a/extensions/Worker.Extensions.CosmosDB/release_notes.md b/extensions/Worker.Extensions.CosmosDB/release_notes.md index d1e4899ce..eb311bd28 100644 --- a/extensions/Worker.Extensions.CosmosDB/release_notes.md +++ b/extensions/Worker.Extensions.CosmosDB/release_notes.md @@ -6,4 +6,6 @@ ### Microsoft.Azure.Functions.Worker.Extensions.CosmosDB -- Add BindingCapabilities attribute to CosmosDb trigger to express function-level retry capabilities. (#1457) +- Add `BindingCapabilities` attribute to CosmosDb trigger to express function-level retry capabilities. (#1457) +- Update `Microsoft.Azure.Cosmos` to `3.34.0` (#1550) +- Updated CosmosDBInputAttribute constructors to allow empty values for databaseName and containerName diff --git a/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs index 21f6446ca..17008e5f7 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs +++ b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs @@ -11,23 +11,26 @@ namespace Microsoft.Azure.Functions.Worker { internal class CosmosDBBindingOptions { + public string? ConnectionName { get; set; } + public string? ConnectionString { get; set; } public string? AccountEndpoint { get; set; } public TokenCredential? Credential { get; set; } - internal string BuildCacheKey(string connectionString, string region) => $"{connectionString}|{region}"; + internal string BuildCacheKey(string connection, string region) => $"{connection}|{region}"; + internal ConcurrentDictionary ClientCache { get; } = new ConcurrentDictionary(); - internal virtual CosmosClient GetClient(string connection, string preferredLocations = "") + internal virtual CosmosClient GetClient(string preferredLocations = "") { - if (string.IsNullOrEmpty(connection)) + if (string.IsNullOrEmpty(ConnectionName)) { - throw new ArgumentNullException(nameof(connection)); + throw new ArgumentNullException(nameof(ConnectionName)); } - string cacheKey = BuildCacheKey(connection, preferredLocations); + string cacheKey = BuildCacheKey(ConnectionName!, preferredLocations); CosmosClientOptions cosmosClientOptions = new () { diff --git a/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptionsSetup.cs b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptionsSetup.cs index 60c2a374a..a7bebfd56 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptionsSetup.cs +++ b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptionsSetup.cs @@ -26,16 +26,18 @@ public void Configure(CosmosDBBindingOptions options) Configure(Options.DefaultName, options); } - public void Configure(string name, CosmosDBBindingOptions options) + public void Configure(string connectionName, CosmosDBBindingOptions options) { - IConfigurationSection connectionSection = _configuration.GetWebJobsConnectionStringSection(name); + IConfigurationSection connectionSection = _configuration.GetWebJobsConnectionStringSection(connectionName); if (!connectionSection.Exists()) { - throw new InvalidOperationException($"Cosmos DB connection configuration '{name}' does not exist. " + + throw new InvalidOperationException($"Cosmos DB connection configuration '{connectionName}' does not exist. " + "Make sure that it is a defined App Setting."); } + options.ConnectionName = connectionName; + if (!string.IsNullOrWhiteSpace(connectionSection.Value)) { options.ConnectionString = connectionSection.Value; diff --git a/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs index 76615a115..cce1ea318 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs +++ b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBConverter.cs @@ -13,11 +13,12 @@ using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; +using Microsoft.Azure.Functions.Worker.Extensions; namespace Microsoft.Azure.Functions.Worker { /// - /// Converter to bind Cosmos DB type parameters. + /// Converter to bind CosmosDB type parameters. /// [SupportsDeferredBinding] internal class CosmosDBConverter : IInputConverter @@ -42,46 +43,35 @@ public async ValueTask ConvertAsync(ConverterContext context) private async ValueTask ConvertFromBindingDataAsync(ConverterContext context, ModelBindingData modelBindingData) { - if (!IsCosmosExtension(modelBindingData)) - { - return ConversionResult.Unhandled(); - } - try { + if (modelBindingData.Source is not Constants.CosmosExtensionName) + { + throw new InvalidBindingSourceException(modelBindingData.Source, Constants.CosmosExtensionName); + } + var cosmosAttribute = GetBindingDataContent(modelBindingData); object result = await ToTargetTypeAsync(context.TargetType, cosmosAttribute); - if (result is not null) - { - return ConversionResult.Success(result); - } + return ConversionResult.Success(result); } catch (Exception ex) { return ConversionResult.Failed(ex); } - - return ConversionResult.Unhandled(); } - private bool IsCosmosExtension(ModelBindingData bindingData) + private CosmosDBInputAttribute GetBindingDataContent(ModelBindingData bindingData) { - if (bindingData?.Source is not Constants.CosmosExtensionName) + if (bindingData is null) { - _logger.LogTrace("Source '{source}' is not supported by {converter}", bindingData?.Source, nameof(CosmosDBConverter)); - return false; + throw new ArgumentNullException(nameof(bindingData)); } - return true; - } - - private CosmosDBInputAttribute GetBindingDataContent(ModelBindingData bindingData) - { - return bindingData?.ContentType switch + return bindingData.ContentType switch { Constants.JsonContentType => bindingData.Content.ToObjectFromJson(), - _ => throw new NotSupportedException($"Unexpected content-type. Currently only '{Constants.JsonContentType}' is supported.") + _ => throw new InvalidContentTypeException(bindingData.ContentType, Constants.JsonContentType) }; } @@ -186,14 +176,19 @@ private async Task> ExtractCosmosDocumentsAsync(FeedIterator iter private T CreateCosmosClient(CosmosDBInputAttribute cosmosAttribute) { - var cosmosDBOptions = _cosmosOptions.Get(cosmosAttribute?.Connection); - CosmosClient cosmosClient = cosmosDBOptions.GetClient(cosmosAttribute?.Connection!, cosmosAttribute?.PreferredLocations!); + if (cosmosAttribute is null) + { + throw new ArgumentNullException(nameof(cosmosAttribute)); + } + + var cosmosDBOptions = _cosmosOptions.Get(cosmosAttribute.Connection); + CosmosClient cosmosClient = cosmosDBOptions.GetClient(cosmosAttribute.PreferredLocations!); Type targetType = typeof(T); object cosmosReference = targetType switch { - Type _ when targetType == typeof(Database) => cosmosClient.GetDatabase(cosmosAttribute?.DatabaseName), - Type _ when targetType == typeof(Container) => cosmosClient.GetContainer(cosmosAttribute?.DatabaseName, cosmosAttribute?.ContainerName), + Type _ when targetType == typeof(Database) => cosmosClient.GetDatabase(cosmosAttribute.DatabaseName), + Type _ when targetType == typeof(Container) => cosmosClient.GetContainer(cosmosAttribute.DatabaseName, cosmosAttribute.ContainerName), _ => cosmosClient }; diff --git a/extensions/Worker.Extensions.CosmosDB/src/CosmosDBInputAttribute.cs b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBInputAttribute.cs index bc6aca573..fe010d330 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/CosmosDBInputAttribute.cs +++ b/extensions/Worker.Extensions.CosmosDB/src/CosmosDBInputAttribute.cs @@ -2,20 +2,44 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System.Collections.Generic; +using System.Text.Json.Serialization; using Microsoft.Azure.Functions.Worker.Converters; using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; namespace Microsoft.Azure.Functions.Worker { - [AllowConverterFallback(false)] [InputConverter(typeof(CosmosDBConverter))] + [ConverterFallbackBehavior(ConverterFallbackBehavior.Default)] public sealed class CosmosDBInputAttribute : InputBindingAttribute { /// /// Constructs a new instance. + /// Use this constructor when binding to a CosmosClient. + /// + public CosmosDBInputAttribute() + { + DatabaseName = string.Empty; + ContainerName = string.Empty; + } + + /// + /// Constructs a new instance with the specified database name. + /// Use this constructor when binding to a Database. + /// + /// The CosmosDB database name. + public CosmosDBInputAttribute(string databaseName) + { + DatabaseName = databaseName; + ContainerName = string.Empty; + } + + /// + /// Constructs a new instance with the specified database and container names. + /// Use this constructor when binding to a Container or a POCO. /// /// The CosmosDB database name. /// The CosmosDB container name. + [JsonConstructor] public CosmosDBInputAttribute(string databaseName, string containerName) { DatabaseName = databaseName; diff --git a/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj b/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj index 1f74029a8..06c047165 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj +++ b/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj @@ -6,7 +6,7 @@ Azure Cosmos DB extensions for .NET isolated functions - 4.3.0 + 4.3.1 -preview1 diff --git a/samples/WorkerBindingSamples/Cosmos/CosmosInputBindingFunctions.cs b/samples/WorkerBindingSamples/Cosmos/CosmosInputBindingFunctions.cs index 437c08bc7..e5b433b0c 100644 --- a/samples/WorkerBindingSamples/Cosmos/CosmosInputBindingFunctions.cs +++ b/samples/WorkerBindingSamples/Cosmos/CosmosInputBindingFunctions.cs @@ -9,6 +9,9 @@ namespace SampleApp { + /// + /// Samples demonstrating binding to the , , and types. + /// public class CosmosInputBindingFunctions { private readonly ILogger _logger; @@ -18,10 +21,15 @@ public CosmosInputBindingFunctions(ILogger logger) _logger = logger; } + /// + /// This sample demonstrates how to retrieve a collection of documents. + /// The code uses a instance to read a list of documents. + /// The instance could also be used for write operations. + /// [Function(nameof(DocsByUsingCosmosClient))] public async Task DocsByUsingCosmosClient( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [CosmosDBInput("", "", Connection = "CosmosDBConnection")] CosmosClient client) + [CosmosDBInput(Connection = "CosmosDBConnection")] CosmosClient client) { _logger.LogInformation("C# HTTP trigger function processed a request."); @@ -40,10 +48,15 @@ public async Task DocsByUsingCosmosClient( return req.CreateResponse(HttpStatusCode.OK); } + /// + /// This sample demonstrates how to retrieve a collection of documents. + /// The function is triggered by an HTTP request and binds to the specified database. + /// as a type. The function then queries for all collections in the database. + /// [Function(nameof(DocsByUsingDatabaseClient))] public async Task DocsByUsingDatabaseClient( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, - [CosmosDBInput("ToDoItems", "", Connection = "CosmosDBConnection")] Database database) + [CosmosDBInput("ToDoItems", Connection = "CosmosDBConnection")] Database database) { _logger.LogInformation("C# HTTP trigger function processed a request."); @@ -61,6 +74,11 @@ public async Task DocsByUsingDatabaseClient( return req.CreateResponse(HttpStatusCode.OK); } + /// + /// This sample demonstrates how to retrieve a collection of documents. + /// The function is triggered by an HTTP request and binds to the specified database and collection + /// as a type. The function then queries for all documents in the collection. + /// [Function(nameof(DocsByUsingContainerClient))] public async Task DocsByUsingContainerClient( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, @@ -82,6 +100,11 @@ public async Task DocsByUsingContainerClient( return req.CreateResponse(HttpStatusCode.OK); } + /// + /// This sample demonstrates how to retrieve a single document. + /// The function is triggered by an HTTP request that uses a query string to specify the ID and partition key value to look up. + /// That ID and partition key value are used to retrieve a ToDoItem document from the specified database and collection. + /// [Function(nameof(DocByIdFromQueryString))] public HttpResponseData DocByIdFromQueryString( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, @@ -106,6 +129,11 @@ public HttpResponseData DocByIdFromQueryString( return req.CreateResponse(HttpStatusCode.OK); } + /// + /// This sample demonstrates how to retrieve a single document. + /// The function is triggered by an HTTP request that uses route data to specify the ID and partition key value to look up. + /// That ID and partition key value are used to retrieve a ToDoItem document from the specified database and collection. + /// [Function(nameof(DocByIdFromRouteData))] public HttpResponseData DocByIdFromRouteData( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "todoitems/{partitionKey}/{id}")] HttpRequestData req, @@ -130,6 +158,12 @@ public HttpResponseData DocByIdFromRouteData( return req.CreateResponse(HttpStatusCode.OK); } + /// + /// This sample demonstrates how to retrieve a collection of documents. + /// The function is triggered by an HTTP request that uses route data to specify the ID to look up. + /// That ID is used to retrieve a list of ToDoItem documents from the specified database and collection. + /// The example shows how to use a binding expression in the parameter. + /// [Function(nameof(DocByIdFromRouteDataUsingSqlQuery))] public HttpResponseData DocByIdFromRouteDataUsingSqlQuery( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "todoitems2/{id}")] HttpRequestData req, @@ -150,6 +184,12 @@ public HttpResponseData DocByIdFromRouteDataUsingSqlQuery( return req.CreateResponse(HttpStatusCode.OK); } + /// + /// This sample demonstrates how to retrieve a collection of documents. + /// The function is triggered by an HTTP request that uses a query string to specify the ID to look up. + /// That ID is used to retrieve a list of ToDoItem documents from the specified database and collection. + /// The example shows how to use a binding expression in the parameter. + /// [Function(nameof(DocByIdFromQueryStringUsingSqlQuery))] public HttpResponseData DocByIdFromQueryStringUsingSqlQuery( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, @@ -170,6 +210,10 @@ public HttpResponseData DocByIdFromQueryStringUsingSqlQuery( return req.CreateResponse(HttpStatusCode.OK); } + /// + /// This sample demonstrates how to retrieve a collection of documents. + /// The function is triggered by an HTTP request. The query is specified in the attribute property. + /// [Function(nameof(DocsBySqlQuery))] public HttpResponseData DocsBySqlQuery( [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, @@ -189,6 +233,12 @@ public HttpResponseData DocsBySqlQuery( return req.CreateResponse(HttpStatusCode.OK); } + /// + /// This sample demonstrates how to retrieve a single document. + /// The function is triggered by a queue message that contains a JSON object. The queue trigger parses the JSON into + /// an object of type ToDoItemLookup, which contains the ID and partition key value to look up. That ID and partition + /// key value are used to retrieve a ToDoItem document from the specified database and collection. + /// [Function(nameof(DocByIdFromJSON))] public void DocByIdFromJSON( [QueueTrigger("todoqueueforlookup")] ToDoItemLookup toDoItemLookup, diff --git a/samples/WorkerBindingSamples/Cosmos/CosmosTriggerFunction.cs b/samples/WorkerBindingSamples/Cosmos/CosmosTriggerFunction.cs index f7e11f6cd..d43086c9d 100644 --- a/samples/WorkerBindingSamples/Cosmos/CosmosTriggerFunction.cs +++ b/samples/WorkerBindingSamples/Cosmos/CosmosTriggerFunction.cs @@ -6,26 +6,35 @@ namespace SampleApp { - // We cannot use SDK-type bindings with the Cosmos trigger binding. There is no way for - // the CosmosDB SDK to let us know the ID of the document that triggered the function; - // therefore we cannot create a client that is able to pull the triggering document. - public static class CosmosTriggerFunction + /// + /// Samples demonstrating binding to the type. + /// + public class CosmosTriggerFunction { + private readonly ILogger _logger; + + public CosmosTriggerFunction(ILogger logger) + { + _logger = logger; + } + + /// + /// This function demonstrates binding to a collection of . + /// [Function(nameof(CosmosTriggerFunction))] - public static void Run([CosmosDBTrigger( + public void Run([CosmosDBTrigger( databaseName: "ToDoItems", containerName:"TriggerItems", Connection = "CosmosDBConnection", + LeaseContainerName = "leases", CreateLeaseContainerIfNotExists = true)] IReadOnlyList todoItems, FunctionContext context) { - var logger = context.GetLogger(nameof(CosmosTriggerFunction)); - if (todoItems is not null && todoItems.Any()) { foreach (var doc in todoItems) { - logger.LogInformation("ToDoItem: {desc}", doc.Description); + _logger.LogInformation("ToDoItem: {desc}", doc.Description); } } } diff --git a/samples/WorkerBindingSamples/local.settings.json b/samples/WorkerBindingSamples/local.settings.json index 401ae0c34..be0735622 100644 --- a/samples/WorkerBindingSamples/local.settings.json +++ b/samples/WorkerBindingSamples/local.settings.json @@ -2,6 +2,8 @@ "IsEncrypted": false, "Values": { "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", - "AzureWebJobsStorage": "UseDevelopmentStorage=true" + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "CosmosDBConnection": "", + "EventHubConnection": "" } } \ No newline at end of file diff --git a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata index fe8e264a7..34a4bc781 100644 --- a/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata +++ b/test/SdkE2ETests/Contents/WorkerBindingSamplesOutput/functions.metadata @@ -315,6 +315,369 @@ } ] }, + { + "name": "DocsByUsingCosmosClient", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsByUsingCosmosClient", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "client", + "direction": "In", + "type": "cosmosDB", + "connection": "CosmosDBConnection", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocsByUsingDatabaseClient", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsByUsingDatabaseClient", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "database", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "connection": "CosmosDBConnection", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocsByUsingContainerClient", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsByUsingContainerClient", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "container", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromQueryString", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromQueryString", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "toDoItem", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "id": "{Query.id}", + "partitionKey": "{Query.partitionKey}", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromRouteData", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromRouteData", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "route": "todoitems/{partitionKey}/{id}", + "properties": {} + }, + { + "name": "toDoItem", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "id": "{id}", + "partitionKey": "{partitionKey}", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromRouteDataUsingSqlQuery", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromRouteDataUsingSqlQuery", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "route": "todoitems2/{id}", + "properties": {} + }, + { + "name": "toDoItems", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "sqlQuery": "SELECT * FROM ToDoItems t where t.id = {id}", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromQueryStringUsingSqlQuery", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromQueryStringUsingSqlQuery", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "toDoItems", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "sqlQuery": "SELECT * FROM ToDoItems t where t.id = {id}", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocsBySqlQuery", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocsBySqlQuery", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "req", + "direction": "In", + "type": "httpTrigger", + "authLevel": "Function", + "methods": [ + "get", + "post" + ], + "properties": {} + }, + { + "name": "toDoItems", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "sqlQuery": "SELECT * FROM ToDoItems t WHERE CONTAINS(t.description, 'cat')", + "properties": { + "supportsDeferredBinding": "True" + } + }, + { + "name": "$return", + "type": "http", + "direction": "Out" + } + ] + }, + { + "name": "DocByIdFromJSON", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosInputBindingFunctions.DocByIdFromJSON", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "toDoItemLookup", + "direction": "In", + "type": "queueTrigger", + "queueName": "todoqueueforlookup", + "properties": {} + }, + { + "name": "toDoItem", + "direction": "In", + "type": "cosmosDB", + "databaseName": "ToDoItems", + "containerName": "Items", + "connection": "CosmosDBConnection", + "id": "{ToDoItemId}", + "partitionKey": "{ToDoItemPartitionKeyValue}", + "properties": { + "supportsDeferredBinding": "True" + } + } + ] + }, + { + "name": "CosmosTriggerFunction", + "scriptFile": "WorkerBindingSamples.dll", + "entryPoint": "SampleApp.CosmosTriggerFunction.Run", + "language": "dotnet-isolated", + "properties": { + "IsCodeless": false + }, + "bindings": [ + { + "name": "todoItems", + "direction": "In", + "type": "cosmosDBTrigger", + "databaseName": "ToDoItems", + "containerName": "TriggerItems", + "connection": "CosmosDBConnection", + "leaseContainerName": "leases", + "createLeaseContainerIfNotExists": true, + "properties": {} + } + ] + }, { "name": "CloudEventFunction", "scriptFile": "WorkerBindingSamples.dll", diff --git a/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs b/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs index 0e1a4e4e8..6d7e91fae 100644 --- a/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs +++ b/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs @@ -31,7 +31,7 @@ public CosmosDBConverterTests() var mockCosmosOptions = new Mock(); mockCosmosOptions - .Setup(m => m.GetClient(It.IsAny(), It.IsAny())) + .Setup(m => m.GetClient(It.IsAny())) .Returns(_mockCosmosClient.Object); var mockCosmosOptionsSnapshot = new Mock>(); @@ -45,7 +45,7 @@ public CosmosDBConverterTests() [Fact] public async Task ConvertAsync_ValidModelBindingData_CosmosClient_ReturnsSuccess() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); var context = new TestConverterContext(typeof(CosmosClient), grpcModelBindingData); _mockCosmosClient.Setup(m => m.Endpoint).Returns(new Uri("https://www.example.com")); @@ -60,7 +60,7 @@ public async Task ConvertAsync_ValidModelBindingData_CosmosClient_ReturnsSuccess [Fact] public async Task ConvertAsync_ValidModelBindingData_DatabaseClient_ReturnsSuccess() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); var context = new TestConverterContext(typeof(Database), grpcModelBindingData); var _mockDatabase = new Mock(); @@ -80,7 +80,7 @@ public async Task ConvertAsync_ValidModelBindingData_DatabaseClient_ReturnsSucce [Fact] public async Task ConvertAsync_ValidModelBindingData_ContainerClient_ReturnsSuccess() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); var context = new TestConverterContext(typeof(Container), grpcModelBindingData); var mockContainer = new Mock(); @@ -100,7 +100,7 @@ public async Task ConvertAsync_ValidModelBindingData_ContainerClient_ReturnsSucc [Fact] public async Task ConvertAsync_ValidModelBindingData_SinglePOCO_ReturnsSuccess() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); var expectedToDoItem = new ToDoItem() { Id = "1", Description = "Take out the rubbish" }; @@ -127,7 +127,7 @@ public async Task ConvertAsync_ValidModelBindingData_SinglePOCO_ReturnsSuccess() [Fact] public async Task ConvertAsync_ValidModelBindingData_SinglePOCO_WithoutId_ReturnsFailed() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(partitionKey: "1"), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(partitionKey: "1"), "CosmosDB"); var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); var mockContainer = new Mock(); @@ -145,7 +145,7 @@ public async Task ConvertAsync_ValidModelBindingData_SinglePOCO_WithoutId_Return [Fact] public async Task ConvertAsync_ValidModelBindingData_SinglePOCO_WithoutPK_ReturnsFailed() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1"), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1"), "CosmosDB"); var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); var mockContainer = new Mock(); @@ -165,7 +165,7 @@ public async Task ConvertAsync_ValidModelBindingData_IEnumerablePOCO_ReturnsSucc { var query = "SELECT * FROM TodoItems t WHERE t.id = @id"; var queryParams = @"{""@id"":""1""}"; - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(query: query, queryParams: queryParams), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(query: query, queryParams: queryParams), "CosmosDB"); var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); var todo1 = new ToDoItem() { Id = "1", Description = "Take out the rubbish" }; @@ -198,7 +198,7 @@ public async Task ConvertAsync_ValidModelBindingData_IEnumerablePOCO_ReturnsSucc [Fact] public async Task ConvertAsync_Container_NullFeedIterator_ReturnsFailed() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); var mockContainer = new Mock(); @@ -222,7 +222,7 @@ public async Task ConvertAsync_ContainerWithSqlQuery_NullFeedIterator_ReturnsFai { var query = "SELECT * FROM TodoItems t WHERE t.id = @id"; var queryParams = @"{""@id"":""1""}"; - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(query: query, queryParams: queryParams), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(query: query, queryParams: queryParams), "CosmosDB"); var context = new TestConverterContext(typeof(IEnumerable), grpcModelBindingData); var mockContainer = new Mock(); @@ -264,7 +264,7 @@ public async Task ConvertAsync_ModelBindingData_Null_ReturnsUnhandled() [Fact] public async Task ConvertAsync_ThrowsException_ReturnsFailure() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB"); var context = new TestConverterContext(typeof(Database), grpcModelBindingData); _mockCosmosClient @@ -279,7 +279,7 @@ public async Task ConvertAsync_ThrowsException_ReturnsFailure() [Fact] public async Task ConvertAsync_ItemResponse_ResourceIsNull_ThrowsException_ReturnsFailed() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); var mockResponse = new Mock>(); @@ -304,30 +304,31 @@ public async Task ConvertAsync_ItemResponse_ResourceIsNull_ThrowsException_Retur [Fact] public async Task ConvertAsync_ModelBindingDataSource_NotCosmosExtension_ReturnsFailed() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "anotherExtensions"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(), "anotherExtensions"); var context = new TestConverterContext(typeof(CosmosClient), grpcModelBindingData); var conversionResult = await _cosmosDBConverter.ConvertAsync(context); Assert.Equal(ConversionStatus.Failed, conversionResult.Status); + Assert.Equal("Unexpected binding source 'anotherExtensions'. Only 'CosmosDB' is supported.", conversionResult.Error.Message); } [Fact] public async Task ConvertAsync_ModelBindingDataContentType_Unsupported_ReturnsFailed() { - var grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB", contentType: "binary"); + var grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(), "CosmosDB", contentType: "binary"); var context = new TestConverterContext(typeof(CosmosClient), grpcModelBindingData); var conversionResult = await _cosmosDBConverter.ConvertAsync(context); Assert.Equal(ConversionStatus.Failed, conversionResult.Status); - Assert.Equal("Unexpected content-type. Only 'application/json' is supported.", conversionResult.Error.Message); + Assert.Equal("Unexpected content-type 'binary'. Only 'application/json' is supported.", conversionResult.Error.Message); } [Fact] public async Task ConvertAsync_CosmosContainerIsNull_ThrowsException_ReturnsFailure() { - object grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(container: "myContainer"), "CosmosDB"); + object grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(container: "myContainer"), "CosmosDB"); var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); _mockCosmosClient @@ -343,7 +344,7 @@ public async Task ConvertAsync_CosmosContainerIsNull_ThrowsException_ReturnsFail [Fact] public async Task ConvertAsync_POCO_ItemResponseNull_ThrowsException_ReturnsFailure() { - object grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); + object grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); var mockContainer = new Mock(); @@ -364,7 +365,7 @@ public async Task ConvertAsync_POCO_ItemResponseNull_ThrowsException_ReturnsFail [Fact] public async Task ConvertAsync_POCO_IdProvided_StatusNot200_ThrowsException_ReturnsFailure() { - object grpcModelBindingData = Helper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); + object grpcModelBindingData = GrpcTestHelper.GetTestGrpcModelBindingData(GetTestBinaryData(id: "1", partitionKey: "1"), "CosmosDB"); var context = new TestConverterContext(typeof(ToDoItem), grpcModelBindingData); var mockResponse = new Mock>(); diff --git a/test/Worker.Extensions.Tests/Cosmos/Helper.cs b/test/Worker.Extensions.Tests/Cosmos/Helper.cs deleted file mode 100644 index 479f793e3..000000000 --- a/test/Worker.Extensions.Tests/Cosmos/Helper.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using Google.Protobuf; -using Microsoft.Azure.Functions.Worker.Grpc.Messages; - -namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Cosmos -{ - public static class Helper - { - internal static GrpcModelBindingData GetTestGrpcModelBindingData(BinaryData content, string source, string contentType = "application/json") - { - var data = new ModelBindingData() - { - Version = "1.0", - Source = source, - Content = ByteString.CopyFrom(content), - ContentType = contentType - }; - - return new GrpcModelBindingData(data); - } - } -} \ No newline at end of file From b62b2098f0b1d4ea304c865d8cd436005a680376 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Thu, 13 Jul 2023 15:02:51 -0700 Subject: [PATCH 42/47] Update CosmosDB version and release notes (#1750) --- extensions/Worker.Extensions.CosmosDB/release_notes.md | 5 ++--- .../src/Worker.Extensions.CosmosDB.csproj | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/extensions/Worker.Extensions.CosmosDB/release_notes.md b/extensions/Worker.Extensions.CosmosDB/release_notes.md index eb311bd28..8f96a86f4 100644 --- a/extensions/Worker.Extensions.CosmosDB/release_notes.md +++ b/extensions/Worker.Extensions.CosmosDB/release_notes.md @@ -4,8 +4,7 @@ - My change description (#PR/#issue) --> -### Microsoft.Azure.Functions.Worker.Extensions.CosmosDB +### Microsoft.Azure.Functions.Worker.Extensions.CosmosDB 4.4.0 -- Add `BindingCapabilities` attribute to CosmosDb trigger to express function-level retry capabilities. (#1457) -- Update `Microsoft.Azure.Cosmos` to `3.34.0` (#1550) - Updated CosmosDBInputAttribute constructors to allow empty values for databaseName and containerName +- Implement input binding support for `CosmosClient`, `Database` and `Container` diff --git a/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj b/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj index 06c047165..4a8cf67af 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj +++ b/extensions/Worker.Extensions.CosmosDB/src/Worker.Extensions.CosmosDB.csproj @@ -6,8 +6,7 @@ Azure Cosmos DB extensions for .NET isolated functions - 4.3.1 - -preview1 + 4.4.0 false From 3ee54aa64ee11198038ed679c4b87f38921abebf Mon Sep 17 00:00:00 2001 From: Brett Samblanet Date: Wed, 26 Jul 2023 07:38:35 -0700 Subject: [PATCH 43/47] removing myget dependency (#1784) --- NuGet.Config | 2 +- samples/FunctionApp/NuGet.Config | 2 +- test/E2ETests/E2EApps/E2EApp/NuGet.Config | 2 +- test/E2ETests/NuGet.Config | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NuGet.Config b/NuGet.Config index 0df60b34c..238ff8b97 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -2,7 +2,7 @@ - + \ No newline at end of file diff --git a/samples/FunctionApp/NuGet.Config b/samples/FunctionApp/NuGet.Config index f187aadea..2a9de739f 100644 --- a/samples/FunctionApp/NuGet.Config +++ b/samples/FunctionApp/NuGet.Config @@ -2,6 +2,6 @@ - + \ No newline at end of file diff --git a/test/E2ETests/E2EApps/E2EApp/NuGet.Config b/test/E2ETests/E2EApps/E2EApp/NuGet.Config index f187aadea..2a9de739f 100644 --- a/test/E2ETests/E2EApps/E2EApp/NuGet.Config +++ b/test/E2ETests/E2EApps/E2EApp/NuGet.Config @@ -2,6 +2,6 @@ - + \ No newline at end of file diff --git a/test/E2ETests/NuGet.Config b/test/E2ETests/NuGet.Config index 0df60b34c..238ff8b97 100644 --- a/test/E2ETests/NuGet.Config +++ b/test/E2ETests/NuGet.Config @@ -2,7 +2,7 @@ - + \ No newline at end of file From 2ebe1775e102cf6005925f503fb58e07b74f81f6 Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 26 Jul 2023 14:03:02 -0700 Subject: [PATCH 44/47] Implement WorkerCosmosSerializer (#1779) --- .../src/Config/CosmosDBBindingOptions.cs | 7 + .../src/CosmosExtensionStartup.cs | 10 +- .../src/WorkerCosmosSerializer.cs | 61 +++++++ .../Cosmos/CosmosDBConverterTests.cs | 1 + .../Cosmos/WorkerCosmosSerializerTests.cs | 159 ++++++++++++++++++ 5 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 extensions/Worker.Extensions.CosmosDB/src/WorkerCosmosSerializer.cs create mode 100644 test/Worker.Extensions.Tests/Cosmos/WorkerCosmosSerializerTests.cs diff --git a/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs index 17008e5f7..8a2eb9e1b 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs +++ b/extensions/Worker.Extensions.CosmosDB/src/Config/CosmosDBBindingOptions.cs @@ -19,6 +19,8 @@ internal class CosmosDBBindingOptions public TokenCredential? Credential { get; set; } + public CosmosSerializer? Serializer { get; set; } + internal string BuildCacheKey(string connection, string region) => $"{connection}|{region}"; internal ConcurrentDictionary ClientCache { get; } = new ConcurrentDictionary(); @@ -42,6 +44,11 @@ internal virtual CosmosClient GetClient(string preferredLocations = "") cosmosClientOptions.ApplicationPreferredRegions = Utilities.ParsePreferredLocations(preferredLocations); } + if (Serializer is not null) + { + cosmosClientOptions.Serializer = Serializer; + } + return ClientCache.GetOrAdd(cacheKey, (c) => CreateService(cosmosClientOptions)); } diff --git a/extensions/Worker.Extensions.CosmosDB/src/CosmosExtensionStartup.cs b/extensions/Worker.Extensions.CosmosDB/src/CosmosExtensionStartup.cs index 5036e3b01..4ee60b670 100644 --- a/extensions/Worker.Extensions.CosmosDB/src/CosmosExtensionStartup.cs +++ b/extensions/Worker.Extensions.CosmosDB/src/CosmosExtensionStartup.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using Microsoft.Azure.Cosmos; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Core; using Microsoft.Extensions.Azure; @@ -22,7 +23,14 @@ public override void Configure(IFunctionsWorkerApplicationBuilder applicationBui } applicationBuilder.Services.AddAzureClientsCore(); // Adds AzureComponentFactory - applicationBuilder.Services.AddOptions(); + + applicationBuilder.Services.AddOptions() + .Configure>((cosmosOptions, workerOptions) => + { + CosmosSerializer cosmosSerializer = new WorkerCosmosSerializer(workerOptions?.Value?.Serializer); + cosmosOptions.Serializer = cosmosSerializer; + }); + applicationBuilder.Services.AddSingleton, CosmosDBBindingOptionsSetup>(); } } diff --git a/extensions/Worker.Extensions.CosmosDB/src/WorkerCosmosSerializer.cs b/extensions/Worker.Extensions.CosmosDB/src/WorkerCosmosSerializer.cs new file mode 100644 index 000000000..480f16d05 --- /dev/null +++ b/extensions/Worker.Extensions.CosmosDB/src/WorkerCosmosSerializer.cs @@ -0,0 +1,61 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +// Adapted from: https://github.com/Azure/azure-cosmos-dotnet-v3/blob/master/Microsoft.Azure.Cosmos/src/Serializer/CosmosJsonDotNetSerializer.cs + +using System; +using System.IO; +using Azure.Core.Serialization; +using Microsoft.Azure.Cosmos; + +namespace Microsoft.Azure.Functions.Worker +{ + /// + /// This is a wrapper that allows us to use the worker options ObjectSerializer to create a CosmosSerializer. + /// + internal sealed class WorkerCosmosSerializer : CosmosSerializer + { + private readonly ObjectSerializer _serializer; + + /// + /// Create a serializer that uses the Azure.Core ObjectSerializer + /// + public WorkerCosmosSerializer(ObjectSerializer? serializer) + { + _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); + } + + /// + /// Convert a Stream to the passed in type. + /// + /// The type of object that should be deserialized. + /// An open stream that is readable that contains JSON. + /// The object representing the deserialized stream. + public override T FromStream(Stream stream) + { + using (stream) + { + if (typeof(Stream).IsAssignableFrom(typeof(T))) + { + return (T)(object)stream; + } + + return (T)_serializer.Deserialize(stream, typeof(T), default)!; + } + } + + /// + /// Converts an object to an open readable stream. + /// + /// The type of object being serialized. + /// The object to be serialized. + /// An open readable stream containing the JSON of the serialized object. + public override Stream ToStream(T input) + { + var streamPayload = new MemoryStream(); + _serializer.Serialize(streamPayload, input, typeof(T), default); + streamPayload.Position = 0; + return streamPayload; + } + } +} \ No newline at end of file diff --git a/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs b/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs index 6d7e91fae..3d843c92b 100644 --- a/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs +++ b/test/Worker.Extensions.Tests/Cosmos/CosmosDBConverterTests.cs @@ -401,6 +401,7 @@ private BinaryData GetTestBinaryData(string db = "testDb", string container = "t return new BinaryData(jsonData); } + public class ToDoItem { public string Id { get; set; } diff --git a/test/Worker.Extensions.Tests/Cosmos/WorkerCosmosSerializerTests.cs b/test/Worker.Extensions.Tests/Cosmos/WorkerCosmosSerializerTests.cs new file mode 100644 index 000000000..49dd97a60 --- /dev/null +++ b/test/Worker.Extensions.Tests/Cosmos/WorkerCosmosSerializerTests.cs @@ -0,0 +1,159 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using Azure.Core.Serialization; +using Xunit; + +namespace Microsoft.Azure.Functions.Worker.Extensions.Tests.Cosmos +{ + public class WorkerCosmosSerializerTests + { + [Fact] + public void FromStream_JsonObjectSerializer_ReturnsObject() + { + // Arrange + var objectSerializer = new JsonObjectSerializer(); + var cosmosSerializer = new WorkerCosmosSerializer(objectSerializer); + + Person person = GetTestPerson(); + var expectedJson = JsonSerializer.Serialize(person); + + using var stream = GenerateStreamFromString(expectedJson); + + // Act + var result = cosmosSerializer.FromStream(stream); + + // Assert + if (expectedJson != JsonSerializer.Serialize(result)) + { + Assert.Fail("Objects are not equal"); + } + } + + [Fact] + public void FromStream_NewtonsoftJsonObjectSerializer_ReturnsObject() + { + // Arrange + var objectSerializer = new NewtonsoftJsonObjectSerializer(); + var cosmosSerializer = new WorkerCosmosSerializer(objectSerializer); + + Person person = GetTestPerson(); + var expectedJson = JsonSerializer.Serialize(person); + + using var stream = GenerateStreamFromString(expectedJson); + + // Act + var result = cosmosSerializer.FromStream(stream); + + // Assert + if (expectedJson != JsonSerializer.Serialize(result)) + { + Assert.Fail("Objects are not equal"); + } + } + + [Fact] + public void ToStream_JsonObjectSerializer_ReturnsStream() + { + // Arrange + var objectSerializer = new JsonObjectSerializer(); + var cosmosSerializer = new WorkerCosmosSerializer(objectSerializer); + + Person person = GetTestPerson(); + + // Act + using var resultStream = cosmosSerializer.ToStream(person); + + StreamReader sr = new StreamReader(resultStream); + var resultPerson = JsonSerializer.Deserialize(sr.ReadToEnd()); + + // Assert + if (JsonSerializer.Serialize(person) != JsonSerializer.Serialize(resultPerson)) + { + Assert.Fail("Objects are not equal"); + } + } + + [Fact] + public void ToStream_NewtonsoftJsonObjectSerializer_ReturnsStream() + { + // Arrange + var objectSerializer = new NewtonsoftJsonObjectSerializer(); + var cosmosSerializer = new WorkerCosmosSerializer(objectSerializer); + + Person person = GetTestPerson(); + + // Act + using var resultStream = cosmosSerializer.ToStream(person); + + StreamReader sr = new StreamReader(resultStream); + var resultPerson = JsonSerializer.Deserialize(sr.ReadToEnd()); + + // Assert + if (JsonSerializer.Serialize(person) != JsonSerializer.Serialize(resultPerson)) + { + Assert.Fail("Objects are not equal"); + } + } + + private static Stream GenerateStreamFromString(string s) + { + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + writer.Write(s); + writer.Flush(); + stream.Position = 0; + return stream; + } + + private static Person GetTestPerson() + { + var children = new List(); + children.Add(new Person + { + Name = "Bob", + City = City.Seattle, + Income = 0, + Children = null, + Age = 5, + Guid = Guid.NewGuid() + }); + + return new Person + { + Name = "Amy", + City = City.Seattle, + Income = 105201, + Children = children, + Age = 35, + Guid = Guid.NewGuid() + }; + } + + private class Person + { + public string Name { get; set; } + + public City City { get; set; } + + public double Income { get; set; } + + public List Children { get; set; } + + public int Age { get; set; } + + public Guid Guid { get; set; } + } + + private enum City + { + NewYork, + LosAngeles, + Seattle + } + } +} \ No newline at end of file From 2d16152a9c9cad8ed71e8aae1f660d7025a9c498 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Wed, 26 Jul 2023 14:12:54 -0700 Subject: [PATCH 45/47] Linux support for FunctionsNetHost (#1704) * FunctionsNetHost in managed code (#1551) First iteration of Azure Functions Custom .NET Host(Validated in Windows) * Adding support for linux cold start (#1594) * Adding support for linux cold start. * Using native "get_hostfxr_path" method from "nethost". Removed PathResolver code (which was our version to do the same thing) * Update build YAML to copy all files instead of just .exe. * Linted yaml file. fixed the copy step to copy only needed dependencies * YAML * Get the NativeApplication instance from pointer and call the methods instead of static method call. * Revert "Get the NativeApplication instance from pointer and call the methods instead of static method call." This reverts commit 2f54a1cdff58d2e47388063f4b029d3b874f36ed. * Using NET8 Preview4. * NET6 support on Debian * Version bump for DotNetIsolatedNativeHost package * Add net6.0;net7.0; to TargetFrameworks. * Removing the shim for Windows. * Rebase on main + used csproj from origin main. Updated dotnetworker.csprok to include net6.0;net7.0; * Removed preivew tag(VersionSuffix) for now. Will add as needed when ready to release the packages. * Nit fixes to address PR feedback(indentation, copyright notice) * Added accidently deleted using statement block * Updating release notes. * Added a global.json for the FunctionsNetHost src directory to use .NET8 since our root level one uses 7.0. We use .NET AOT compiler to publish the FunctionsNetHost. * Fixed the version to 8.0.100-preview * Added "includePreviewVersions: true" to "Install current .NET SDK" build step. * Switching version to 8.0.100-preview.6 and "rollForward" value to "latestMinor" in FunctionsNetHost/src/global.json * Specifically set 7.0.306 in Install current .NET SDK step * Add a new UseDotnet task to install .net8 preview 6 in install-dotnet.yml * set includePreviewVersions true for new step * usesGlobalJson false for new UseDotnet task * use specific version 8.0.100-preview.6.23330.14 --- host/azure-pipelines.yml | 162 ++++++++++---- host/src/.clang-format | 2 - host/src/CMakeLists.txt | 28 --- host/src/CMakePresets.json | 64 ------ host/src/FunctionsNetHost.sln | 25 +++ .../FunctionsNetHost/AppLoader/AppLoader.cs | 107 ++++++++++ .../src/FunctionsNetHost/AppLoader/HostFxr.cs | 50 +++++ .../src/FunctionsNetHost/AppLoader/NetHost.cs | 31 +++ .../WorkerLoadStatusSignalManager.cs | 19 ++ host/src/FunctionsNetHost/CMakeLists.txt | 20 -- .../Environment/EnvironmentSettingNames.cs | 12 ++ .../Environment/EnvironmentUtils.cs | 37 ++++ .../FunctionsNetHost/FunctionsNetHost.csproj | 39 ++++ host/src/FunctionsNetHost/Grpc/GrpcClient.cs | 134 ++++++++++++ .../Grpc/GrpcWorkerStartupOptions.cs | 18 ++ .../Grpc/IncomingGrpcMessageHandler.cs | 101 +++++++++ .../FunctionsNetHost/Grpc/MessageChannel.cs | 61 ++++++ host/src/FunctionsNetHost/Grpc/PathUtils.cs | 53 +++++ host/src/FunctionsNetHost/Logger.cs | 46 ++++ .../FunctionsNetHost/Native/NativeExports.cs | 71 +++++++ .../Native/NativeHostApplication.cs | 32 +++ .../FunctionsNetHost/Native/NativeHostData.cs | 9 + host/src/FunctionsNetHost/Program.cs | 66 ++++++ .../Properties/launchSettings.json | 8 + host/src/FunctionsNetHost/exports.def | 4 + host/src/FunctionsNetHost/global.json | 7 + host/src/FunctionsNetHost/main.cpp | 89 -------- host/src/FunctionsNetHost/managedexports.cpp | 58 ----- host/src/Protos/CMakeLists.txt | 44 ---- host/src/README.md | 12 +- host/src/funcgrpc/CMakeLists.txt | 41 ---- host/src/funcgrpc/byte_buffer_helper.cpp | 49 ----- host/src/funcgrpc/byte_buffer_helper.h | 23 -- host/src/funcgrpc/func_bidi_reactor.cpp | 201 ------------------ host/src/funcgrpc/func_bidi_reactor.h | 110 ---------- host/src/funcgrpc/func_log.cpp | 22 -- host/src/funcgrpc/func_log.h | 35 --- host/src/funcgrpc/func_perf_marker.h | 35 --- host/src/funcgrpc/funcgrpc.h | 29 --- host/src/funcgrpc/funcgrpc_handlers.h | 28 --- .../funcgrpc_worker_config_handle.cpp | 52 ----- .../funcgrpc/funcgrpc_worker_config_handle.h | 27 --- .../handlers/funcgrpc_native_handler.cpp | 114 ---------- .../handlers/funcgrpc_native_handler.h | 38 ---- host/src/funcgrpc/messaging_channel.cpp | 27 --- host/src/funcgrpc/messaging_channel.h | 51 ----- host/src/funcgrpc/nativehostapplication.cpp | 119 ----------- host/src/funcgrpc/nativehostapplication.h | 70 ------ host/src/vcpkg.json | 14 -- ....Functions.DotnetIsolatedNativeHost.nuspec | 5 +- host/tools/build/worker.config.json | 18 +- release_notes.md | 2 +- .../DotNetWorker.Grpc.csproj | 7 +- .../NativeHostIntegration/NativeMethods.cs | 46 +++- .../Shim/NativeLibrary.Linux.cs | 49 +++++ src/DotNetWorker/DotNetWorker.csproj | 2 +- 56 files changed, 1172 insertions(+), 1451 deletions(-) delete mode 100644 host/src/.clang-format delete mode 100644 host/src/CMakeLists.txt delete mode 100644 host/src/CMakePresets.json create mode 100644 host/src/FunctionsNetHost.sln create mode 100644 host/src/FunctionsNetHost/AppLoader/AppLoader.cs create mode 100644 host/src/FunctionsNetHost/AppLoader/HostFxr.cs create mode 100644 host/src/FunctionsNetHost/AppLoader/NetHost.cs create mode 100644 host/src/FunctionsNetHost/AppLoader/WorkerLoadStatusSignalManager.cs delete mode 100644 host/src/FunctionsNetHost/CMakeLists.txt create mode 100644 host/src/FunctionsNetHost/Environment/EnvironmentSettingNames.cs create mode 100644 host/src/FunctionsNetHost/Environment/EnvironmentUtils.cs create mode 100644 host/src/FunctionsNetHost/FunctionsNetHost.csproj create mode 100644 host/src/FunctionsNetHost/Grpc/GrpcClient.cs create mode 100644 host/src/FunctionsNetHost/Grpc/GrpcWorkerStartupOptions.cs create mode 100644 host/src/FunctionsNetHost/Grpc/IncomingGrpcMessageHandler.cs create mode 100644 host/src/FunctionsNetHost/Grpc/MessageChannel.cs create mode 100644 host/src/FunctionsNetHost/Grpc/PathUtils.cs create mode 100644 host/src/FunctionsNetHost/Logger.cs create mode 100644 host/src/FunctionsNetHost/Native/NativeExports.cs create mode 100644 host/src/FunctionsNetHost/Native/NativeHostApplication.cs create mode 100644 host/src/FunctionsNetHost/Native/NativeHostData.cs create mode 100644 host/src/FunctionsNetHost/Program.cs create mode 100644 host/src/FunctionsNetHost/Properties/launchSettings.json create mode 100644 host/src/FunctionsNetHost/exports.def create mode 100644 host/src/FunctionsNetHost/global.json delete mode 100644 host/src/FunctionsNetHost/main.cpp delete mode 100644 host/src/FunctionsNetHost/managedexports.cpp delete mode 100644 host/src/Protos/CMakeLists.txt delete mode 100644 host/src/funcgrpc/CMakeLists.txt delete mode 100644 host/src/funcgrpc/byte_buffer_helper.cpp delete mode 100644 host/src/funcgrpc/byte_buffer_helper.h delete mode 100644 host/src/funcgrpc/func_bidi_reactor.cpp delete mode 100644 host/src/funcgrpc/func_bidi_reactor.h delete mode 100644 host/src/funcgrpc/func_log.cpp delete mode 100644 host/src/funcgrpc/func_log.h delete mode 100644 host/src/funcgrpc/func_perf_marker.h delete mode 100644 host/src/funcgrpc/funcgrpc.h delete mode 100644 host/src/funcgrpc/funcgrpc_handlers.h delete mode 100644 host/src/funcgrpc/funcgrpc_worker_config_handle.cpp delete mode 100644 host/src/funcgrpc/funcgrpc_worker_config_handle.h delete mode 100644 host/src/funcgrpc/handlers/funcgrpc_native_handler.cpp delete mode 100644 host/src/funcgrpc/handlers/funcgrpc_native_handler.h delete mode 100644 host/src/funcgrpc/messaging_channel.cpp delete mode 100644 host/src/funcgrpc/messaging_channel.h delete mode 100644 host/src/funcgrpc/nativehostapplication.cpp delete mode 100644 host/src/funcgrpc/nativehostapplication.h delete mode 100644 host/src/vcpkg.json create mode 100644 src/DotNetWorker.Grpc/NativeHostIntegration/Shim/NativeLibrary.Linux.cs diff --git a/host/azure-pipelines.yml b/host/azure-pipelines.yml index 2eb7fec2b..deb7d5476 100644 --- a/host/azure-pipelines.yml +++ b/host/azure-pipelines.yml @@ -1,14 +1,9 @@ -variables: - - name: VCPKG_BINARY_SOURCES - value: "clear;nuget,https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/FunctionsNetHostBinaryCache/nuget/v3/index.json,readwrite" - - name: VCPKG_USE_NUGET_CACHE - value: 1 - trigger: branches: include: - main - release/* + - feature/* paths: include: - host/src/ @@ -17,38 +12,131 @@ pr: include: - main - release/* + - feature/* paths: include: - host/src/ +stages: + - stage: BuildAndPublish + displayName: "Dotnet Publish(W+L)" + jobs: + - job: BuildAndPublishLinux + displayName: "Publish on Linux" + pool: + vmImage: "ubuntu-20.04" + steps: + - task: UseDotNet@2 + inputs: + version: "8.x" + includePreviewVersions: true + + - script: | + sudo apt-get install clang zlib1g-dev + + - task: DotnetCoreCLI@2 + displayName: "Dotnet Publish" + inputs: + command: "publish" + publishWebProjects: false + zipAfterPublish: false + arguments: "-c Release -r linux-x64 -o $(Build.ArtifactStagingDirectory)/output/linux" + workingDirectory: $(Build.SourcesDirectory)/host/src/FunctionsNetHost + + - task: CopyFiles@2 + displayName: "Copy needed files" + inputs: + SourceFolder: "$(Build.ArtifactStagingDirectory)/output/linux" + # Publish output will include many other files. We only need the FunctionsNetHost & libnethost.so + Contents: | + FunctionsNetHost + libnethost.so + TargetFolder: "$(Build.ArtifactStagingDirectory)/output/linux_filtered" + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: "$(Build.ArtifactStagingDirectory)/output/linux_filtered" + artifact: "linux_publish_output" + + - job: BuildAndPublishWindows + displayName: "Publish on Windows" + pool: + vmImage: "windows-latest" + steps: + - task: UseDotNet@2 + inputs: + version: "8.x" + includePreviewVersions: true + + - task: DotnetCoreCLI@2 + displayName: "Dotnet Publish" + inputs: + command: "publish" + publishWebProjects: false + zipAfterPublish: false + arguments: "-c Release -r win-x64 -o $(Build.ArtifactStagingDirectory)/output/windows" + workingDirectory: $(Build.SourcesDirectory)/host/src/FunctionsNetHost + + - task: CopyFiles@2 + displayName: "Copy needed files" + inputs: + SourceFolder: "$(Build.ArtifactStagingDirectory)/output/windows" + # Publish output will include many other files. We only need FunctionsNetHost.exe & nethost.dll + Contents: | + FunctionsNetHost.exe + nethost.dll + TargetFolder: "$(Build.ArtifactStagingDirectory)/output/windows_filtered" + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: "$(Build.ArtifactStagingDirectory)/output/windows_filtered" + artifact: "windows_publish_output" + + - stage: ConsolidateArtifacts + displayName: "Nuget Publish" + dependsOn: BuildAndPublish + jobs: + - job: ConsolidateArtifacts + displayName: "Consolidate Artifacts" + pool: + vmImage: "windows-latest" + steps: + - task: UseDotNet@2 + inputs: + version: "7.x" + + - task: DownloadPipelineArtifact@2 + displayName: "Download Artifacts from Linux build" + inputs: + artifactName: "linux_publish_output" + path: "$(Build.ArtifactStagingDirectory)/linux_output" + + - task: DownloadPipelineArtifact@2 + displayName: "Download Artifacts from Windows build" + inputs: + artifactName: "windows_publish_output" + path: "$(Build.ArtifactStagingDirectory)/windows_output" + + - task: CopyFiles@2 + displayName: "Copy files from linux artifacts to dist dir" + inputs: + SourceFolder: "$(Build.ArtifactStagingDirectory)/linux_output" + TargetFolder: "$(Build.SourcesDirectory)/host/dist/linux" + + - task: CopyFiles@2 + displayName: "Copy files from Windows artifacts to dist dir" + inputs: + SourceFolder: "$(Build.ArtifactStagingDirectory)/windows_output" + TargetFolder: "$(Build.SourcesDirectory)/host/dist/windows" + + - task: NuGetCommand@2 + displayName: "Nuget pack" + inputs: + command: "pack" + packagesToPack: "$(Build.SourcesDirectory)/host/tools/build/Microsoft.Azure.Functions.DotnetIsolatedNativeHost.nuspec" + versioningScheme: "off" + packDestination: "$(Build.ArtifactStagingDirectory)/host/dist/nuget" + basePath: "$(Build.SourcesDirectory)/host/tools/build" -pool: - vmImage: windows-latest - -steps: - # Remember to add this task to allow vcpkg to upload archives via NuGet - - task: NuGetAuthenticate@1 - - # Run Cmake and output goes to /build dir. - - task: CMake@1 - displayName: "CMake generation" - inputs: - cmakeArgs: "-S $(Build.SourcesDirectory)/host/src -B $(Build.SourcesDirectory)/host/build/win-x64" - - # Run Cmake --build which produces the native binaries. - - task: CMake@1 - displayName: "CMake build" - inputs: - cmakeArgs: "--build $(Build.SourcesDirectory)/host/build/win-x64 --config Release" - - - task: NuGetCommand@2 - displayName: "Nuget pack" - inputs: - command: "pack" - packagesToPack: "$(Build.SourcesDirectory)/host/tools/build/Microsoft.Azure.Functions.DotnetIsolatedNativeHost.nuspec" - versioningScheme: "off" - packDestination: "$(Build.ArtifactStagingDirectory)/host/build/nuget" - basePath: "$(Build.SourcesDirectory)/host/tools/build" - - # Publish artifacts. - - publish: $(Build.ArtifactStagingDirectory)/host/build/nuget - artifact: drop + # Publish artifacts. + - publish: $(Build.ArtifactStagingDirectory)/host/dist/nuget + artifact: drop diff --git a/host/src/.clang-format b/host/src/.clang-format deleted file mode 100644 index 2df4c36a4..000000000 --- a/host/src/.clang-format +++ /dev/null @@ -1,2 +0,0 @@ -Language: Cpp -BasedOnStyle: Microsoft \ No newline at end of file diff --git a/host/src/CMakeLists.txt b/host/src/CMakeLists.txt deleted file mode 100644 index 4612ba95c..000000000 --- a/host/src/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# CMakeList.txt : Top-level CMake project file, do global configuration -# and include sub-projects here. -# -cmake_minimum_required (VERSION 3.23) - -include(FetchContent) -FetchContent_Declare(vcpkg - GIT_REPOSITORY https://github.com/microsoft/vcpkg/ - GIT_TAG 2022.11.14 -) -FetchContent_MakeAvailable(vcpkg) - -# NOTE: This must be defined before the first project call -set(CMAKE_TOOLCHAIN_FILE "${vcpkg_SOURCE_DIR}/scripts/buildsystems/vcpkg.cmake" CACHE FILEPATH "") - -set(VCPKG_INSTALL_OPTIONS "--debug") - -project ("FunctionsNetHost") -set(CMAKE_CXX_STANDARD 20) - -# Include sub-projects. -add_subdirectory("Protos") -add_subdirectory ("FunctionsNetHost") -add_subdirectory("funcgrpc") - -target_include_directories(FunctionsNetHost PUBLIC - "${PROJECT_BINARY_DIR}" - ) \ No newline at end of file diff --git a/host/src/CMakePresets.json b/host/src/CMakePresets.json deleted file mode 100644 index 08bb7a625..000000000 --- a/host/src/CMakePresets.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "version": 3, - "configurePresets": [ - { - "name": "windows-base", - "hidden": true, - "generator": "Ninja", - "binaryDir": "${sourceDir}/out/build/${presetName}", - "installDir": "${sourceDir}/out/install/${presetName}", - "cacheVariables": { - "CMAKE_C_COMPILER": "cl.exe", - "CMAKE_CXX_COMPILER": "cl.exe" - }, - "condition": { - "type": "equals", - "lhs": "${hostSystemName}", - "rhs": "Windows" - } - }, - { - "name": "x64-debug", - "displayName": "x64 Debug", - "inherits": "windows-base", - "architecture": { - "value": "x64", - "strategy": "external" - }, - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - }, - "environment": { - "Test": "test" - } - }, - { - "name": "x64-release", - "displayName": "x64 Release", - "inherits": "x64-debug", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } - }, - { - "name": "x86-debug", - "displayName": "x86 Debug", - "inherits": "windows-base", - "architecture": { - "value": "x86", - "strategy": "external" - }, - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - } - }, - { - "name": "x86-release", - "displayName": "x86 Release", - "inherits": "x86-debug", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } - } - ] -} diff --git a/host/src/FunctionsNetHost.sln b/host/src/FunctionsNetHost.sln new file mode 100644 index 000000000..4e8b85966 --- /dev/null +++ b/host/src/FunctionsNetHost.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.33627.172 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FunctionsNetHost", "FunctionsNetHost\FunctionsNetHost.csproj", "{6C05D0AC-F6AC-45FB-8A73-A3F44DF131BC}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {6C05D0AC-F6AC-45FB-8A73-A3F44DF131BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6C05D0AC-F6AC-45FB-8A73-A3F44DF131BC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6C05D0AC-F6AC-45FB-8A73-A3F44DF131BC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6C05D0AC-F6AC-45FB-8A73-A3F44DF131BC}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {32F98336-4B56-47A5-806E-0D1CC5F9F48B} + EndGlobalSection +EndGlobal diff --git a/host/src/FunctionsNetHost/AppLoader/AppLoader.cs b/host/src/FunctionsNetHost/AppLoader/AppLoader.cs new file mode 100644 index 000000000..fd3005688 --- /dev/null +++ b/host/src/FunctionsNetHost/AppLoader/AppLoader.cs @@ -0,0 +1,107 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Runtime.InteropServices; + +namespace FunctionsNetHost +{ + /// + /// Manages loading hostfxr & worker assembly. + /// + internal sealed class AppLoader : IDisposable + { + private IntPtr _hostfxrHandle = IntPtr.Zero; + private IntPtr _hostContextHandle = IntPtr.Zero; + private bool _disposed; + + internal AppLoader() + { + LoadHostfxrLibrary(); + } + + private void LoadHostfxrLibrary() + { + // If having problems with the managed host, enable the following: + // Environment.SetEnvironmentVariable("COREHOST_TRACE", "1"); + // In Unix environment, you need to run the below command in the terminal to set the environment variable. + // export COREHOST_TRACE=1 + + var hostfxrFullPath = NetHost.GetHostFxrPath(); + Logger.LogTrace($"hostfxr path:{hostfxrFullPath}"); + + _hostfxrHandle = NativeLibrary.Load(hostfxrFullPath); + if (_hostfxrHandle == IntPtr.Zero) + { + Logger.Log($"Failed to load hostfxr. hostfxr path:{hostfxrFullPath}"); + return; + } + + Logger.LogTrace($"hostfxr library loaded successfully."); + } + + internal int RunApplication(string? assemblyPath) + { + ArgumentNullException.ThrowIfNull(assemblyPath, nameof(assemblyPath)); + + unsafe + { + var parameters = new HostFxr.hostfxr_initialize_parameters + { + size = sizeof(HostFxr.hostfxr_initialize_parameters) + }; + + var error = HostFxr.Initialize(1, new[] { assemblyPath }, ref parameters, out _hostContextHandle); + + if (_hostContextHandle == IntPtr.Zero) + { + Logger.Log( + $"Failed to initialize the .NET Core runtime. Assembly path:{assemblyPath}"); + return -1; + } + + if (error < 0) + { + return error; + } + + Logger.LogTrace($"hostfxr initialized with {assemblyPath}"); + HostFxr.SetAppContextData(_hostContextHandle, "AZURE_FUNCTIONS_NATIVE_HOST", "1"); + + return HostFxr.Run(_hostContextHandle); + } + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool disposing) + { + if (!_disposed) + { + if (!disposing) + { + return; + } + + if (_hostfxrHandle != IntPtr.Zero) + { + NativeLibrary.Free(_hostfxrHandle); + Logger.LogTrace($"Freed hostfxr library handle"); + _hostfxrHandle = IntPtr.Zero; + } + + if (_hostContextHandle != IntPtr.Zero) + { + NativeLibrary.Free(_hostContextHandle); + Logger.LogTrace($"Freed hostcontext handle"); + _hostContextHandle = IntPtr.Zero; + } + + _disposed = true; + } + } + } +} diff --git a/host/src/FunctionsNetHost/AppLoader/HostFxr.cs b/host/src/FunctionsNetHost/AppLoader/HostFxr.cs new file mode 100644 index 000000000..7655ca919 --- /dev/null +++ b/host/src/FunctionsNetHost/AppLoader/HostFxr.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Runtime.InteropServices; + +namespace FunctionsNetHost +{ + static partial class HostFxr + { + public unsafe struct hostfxr_initialize_parameters + { + public nint size; + public char* host_path; + public char* dotnet_root; + }; + + [LibraryImport("hostfxr", EntryPoint = "hostfxr_initialize_for_dotnet_command_line")] + public unsafe static partial int Initialize( + int argc, + [MarshalAs(UnmanagedType.LPArray, ArraySubType = +#if OS_LINUX + UnmanagedType.LPStr +#else + UnmanagedType.LPWStr +#endif + )] string[] argv, + ref hostfxr_initialize_parameters parameters, + out IntPtr host_context_handle + ); + + [LibraryImport("hostfxr", EntryPoint = "hostfxr_run_app")] + public static partial int Run(IntPtr host_context_handle); + + [LibraryImport("hostfxr", EntryPoint = "hostfxr_set_runtime_property_value")] + public static partial int SetAppContextData(IntPtr host_context_handle, [MarshalAs( +#if OS_LINUX + UnmanagedType.LPStr +#else + UnmanagedType.LPWStr +#endif + )] string name, [MarshalAs( +#if OS_LINUX + UnmanagedType.LPStr +#else + UnmanagedType.LPWStr +#endif + )] string value); + + } +} diff --git a/host/src/FunctionsNetHost/AppLoader/NetHost.cs b/host/src/FunctionsNetHost/AppLoader/NetHost.cs new file mode 100644 index 000000000..61397c29f --- /dev/null +++ b/host/src/FunctionsNetHost/AppLoader/NetHost.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Runtime.InteropServices; + +namespace FunctionsNetHost +{ + internal class NetHost + { + [DllImport("nethost", CharSet = CharSet.Auto)] + private static extern int get_hostfxr_path( + [Out] char[] buffer, + [In] ref int buffer_size, + IntPtr reserved); + + internal static string GetHostFxrPath() + { + char[] buffer = new char[200]; + int bufferSize = buffer.Length; + + int rc = get_hostfxr_path(buffer, ref bufferSize, IntPtr.Zero); + + if (rc != 0) + { + throw new InvalidOperationException("Failed to get the hostfxr path."); + } + + return new string(buffer, 0, bufferSize - 1); + } + } +} diff --git a/host/src/FunctionsNetHost/AppLoader/WorkerLoadStatusSignalManager.cs b/host/src/FunctionsNetHost/AppLoader/WorkerLoadStatusSignalManager.cs new file mode 100644 index 000000000..6bd3f9e51 --- /dev/null +++ b/host/src/FunctionsNetHost/AppLoader/WorkerLoadStatusSignalManager.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace FunctionsNetHost; + +/// +/// Provides a signaling mechanism to wait and get notified about successful load of worker assembly. +/// +public class WorkerLoadStatusSignalManager +{ + private WorkerLoadStatusSignalManager() + { + Signal = new ManualResetEvent(false); + } + + public static WorkerLoadStatusSignalManager Instance { get; } = new(); + + public readonly ManualResetEvent Signal; +} diff --git a/host/src/FunctionsNetHost/CMakeLists.txt b/host/src/FunctionsNetHost/CMakeLists.txt deleted file mode 100644 index 6ada4967d..000000000 --- a/host/src/FunctionsNetHost/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# CMakeList.txt : CMake project for FunctionsNetHost, include source and define -# project specific logic here. -# -add_executable (FunctionsNetHost "main.cpp" "managedexports.cpp") -target_link_libraries(FunctionsNetHost PRIVATE funcgrpc) - -set_target_properties(FunctionsNetHost PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) - -if (CMAKE_VERSION VERSION_GREATER 3.12) - set_property(TARGET FunctionsNetHost PROPERTY CXX_STANDARD 20) -endif() - -find_package(Boost REQUIRED COMPONENTS program_options) -target_link_libraries(FunctionsNetHost PRIVATE Boost::program_options) - -if(NOT TARGET spdlog) - # Stand-alone build - find_package(spdlog REQUIRED) -endif() -target_link_libraries(FunctionsNetHost PRIVATE spdlog::spdlog) \ No newline at end of file diff --git a/host/src/FunctionsNetHost/Environment/EnvironmentSettingNames.cs b/host/src/FunctionsNetHost/Environment/EnvironmentSettingNames.cs new file mode 100644 index 000000000..f23da721b --- /dev/null +++ b/host/src/FunctionsNetHost/Environment/EnvironmentSettingNames.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace FunctionsNetHost; + +internal static class EnvironmentSettingNames +{ + /// + /// Set value to "1" for enabling extra trace logs in FunctionsNetHost. + /// + internal const string FunctionsNetHostTrace = "AZURE_FUNCTIONS_FUNCTIONSNETHOST_TRACE"; +} diff --git a/host/src/FunctionsNetHost/Environment/EnvironmentUtils.cs b/host/src/FunctionsNetHost/Environment/EnvironmentUtils.cs new file mode 100644 index 000000000..024a99315 --- /dev/null +++ b/host/src/FunctionsNetHost/Environment/EnvironmentUtils.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace FunctionsNetHost +{ + internal static class EnvironmentUtils + { +#if OS_LINUX + [System.Runtime.InteropServices.DllImport("libc")] + private static extern int setenv(string name, string value, int overwrite); +#endif + + /// + /// Gets the environment variable value. + /// + internal static string? GetValue(string environmentVariableName) + { + return Environment.GetEnvironmentVariable(environmentVariableName); + } + + /// + /// Sets the environment variable value. + /// + internal static void SetValue(string name, string value) + { + /* + * Environment.SetEnvironmentVariable is not setting the value of the parent process in Unix. + * So using the native method directly here. + * */ +#if OS_LINUX + setenv(name, value, 1); +#else + Environment.SetEnvironmentVariable(name, value); +#endif + } + } +} diff --git a/host/src/FunctionsNetHost/FunctionsNetHost.csproj b/host/src/FunctionsNetHost/FunctionsNetHost.csproj new file mode 100644 index 000000000..9a761b186 --- /dev/null +++ b/host/src/FunctionsNetHost/FunctionsNetHost.csproj @@ -0,0 +1,39 @@ + + + + Exe + net8.0 + enable + enable + True + true + Speed + true + + + + OS_LINUX + + + + $(MSBuildThisFileDirectory)exports.def + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + diff --git a/host/src/FunctionsNetHost/Grpc/GrpcClient.cs b/host/src/FunctionsNetHost/Grpc/GrpcClient.cs new file mode 100644 index 000000000..bbc1f7167 --- /dev/null +++ b/host/src/FunctionsNetHost/Grpc/GrpcClient.cs @@ -0,0 +1,134 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Threading.Channels; +using Google.Protobuf; +using Grpc.Core; +using Grpc.Net.Client; +using Microsoft.Azure.Functions.Worker.Grpc.Messages; +using static Microsoft.Azure.Functions.Worker.Grpc.Messages.FunctionRpc; + +namespace FunctionsNetHost.Grpc +{ + internal sealed class GrpcClient + { + private readonly Channel _outgoingMessageChannel; + private readonly IncomingGrpcMessageHandler _messageHandler; + private readonly GrpcWorkerStartupOptions _grpcWorkerStartupOptions; + + internal GrpcClient(GrpcWorkerStartupOptions grpcWorkerStartupOptions, AppLoader appLoader) + { + _grpcWorkerStartupOptions = grpcWorkerStartupOptions; + var channelOptions = new UnboundedChannelOptions + { + SingleWriter = false, + SingleReader = false, + AllowSynchronousContinuations = true + }; + + _outgoingMessageChannel = Channel.CreateUnbounded(channelOptions); + + _messageHandler = new IncomingGrpcMessageHandler(appLoader); + } + + internal async Task InitAsync() + { + var endpoint = $"http://{_grpcWorkerStartupOptions.Host}:{_grpcWorkerStartupOptions.Port}"; + Logger.LogTrace($"Grpc service endpoint:{endpoint}"); + + var functionRpcClient = CreateFunctionRpcClient(endpoint); + var eventStream = functionRpcClient.EventStream(); + + await SendStartStreamMessageAsync(eventStream.RequestStream); + + var readerTask = StartReaderAsync(eventStream.ResponseStream); + var writerTask = StartWriterAsync(eventStream.RequestStream); + _ = StartInboundMessageForwarding(); + _ = StartOutboundMessageForwarding(); + + await Task.WhenAll(readerTask, writerTask); + } + + private async Task StartReaderAsync(IAsyncStreamReader responseStream) + { + while (await responseStream.MoveNext()) + { + await _messageHandler.ProcessMessageAsync(responseStream.Current); + } + } + + private async Task StartWriterAsync(IClientStreamWriter requestStream) + { + await foreach (var rpcWriteMsg in _outgoingMessageChannel.Reader.ReadAllAsync()) + { + await requestStream.WriteAsync(rpcWriteMsg); + } + } + + private async Task SendStartStreamMessageAsync(IClientStreamWriter requestStream) + { + var startStreamMsg = new StartStream() + { + WorkerId = _grpcWorkerStartupOptions.WorkerId + }; + + var startStream = new StreamingMessage() + { + StartStream = startStreamMsg + }; + + await requestStream.WriteAsync(startStream); + } + + private FunctionRpcClient CreateFunctionRpcClient(string endpoint) + { + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out var grpcUri)) + { + throw new InvalidOperationException($"The gRPC channel URI '{endpoint}' could not be parsed."); + } + + var grpcChannel = GrpcChannel.ForAddress(grpcUri, new GrpcChannelOptions() + { + MaxReceiveMessageSize = _grpcWorkerStartupOptions.GrpcMaxMessageLength, + MaxSendMessageSize = _grpcWorkerStartupOptions.GrpcMaxMessageLength, + Credentials = ChannelCredentials.Insecure + }); + + return new FunctionRpcClient(grpcChannel); + } + + /// + /// Listens to messages in the inbound message channel and forward them the customer payload via interop layer. + /// + private async Task StartInboundMessageForwarding() + { + await foreach (var inboundMessage in MessageChannel.Instance.InboundChannel.Reader.ReadAllAsync()) + { + await HandleIncomingMessage(inboundMessage); + } + } + + /// + /// Listens to messages in the inbound message channel and forward them the customer payload via interop layer. + /// + private async Task StartOutboundMessageForwarding() + { + await foreach (var outboundMessage in MessageChannel.Instance.OutboundChannel.Reader.ReadAllAsync()) + { + await _outgoingMessageChannel.Writer.WriteAsync(outboundMessage); + } + } + + private static Task HandleIncomingMessage(StreamingMessage inboundMessage) + { + // Queue the work to another thread. + Task.Run(() => + { + byte[] inboundMessageBytes = inboundMessage.ToByteArray(); + NativeHostApplication.Instance.HandleInboundMessage(inboundMessageBytes, inboundMessageBytes.Length); + }); + + return Task.CompletedTask; + } + } +} diff --git a/host/src/FunctionsNetHost/Grpc/GrpcWorkerStartupOptions.cs b/host/src/FunctionsNetHost/Grpc/GrpcWorkerStartupOptions.cs new file mode 100644 index 000000000..3ecf513b8 --- /dev/null +++ b/host/src/FunctionsNetHost/Grpc/GrpcWorkerStartupOptions.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace FunctionsNetHost.Grpc +{ + internal sealed class GrpcWorkerStartupOptions + { + public string? Host { get; set; } + + public int Port { get; set; } + + public string? WorkerId { get; set; } + + public string? RequestId { get; set; } + + public int GrpcMaxMessageLength { get; set; } + } +} diff --git a/host/src/FunctionsNetHost/Grpc/IncomingGrpcMessageHandler.cs b/host/src/FunctionsNetHost/Grpc/IncomingGrpcMessageHandler.cs new file mode 100644 index 000000000..4beb947ad --- /dev/null +++ b/host/src/FunctionsNetHost/Grpc/IncomingGrpcMessageHandler.cs @@ -0,0 +1,101 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.Functions.Worker.Grpc.Messages; + +namespace FunctionsNetHost.Grpc +{ + internal sealed class IncomingGrpcMessageHandler + { + private bool _specializationDone; + private readonly AppLoader _appLoader; + + internal IncomingGrpcMessageHandler(AppLoader appLoader) + { + _appLoader = appLoader; + } + + internal Task ProcessMessageAsync(StreamingMessage message) + { + Task.Run(() => Process(message)); + + return Task.CompletedTask; + } + + private async Task Process(StreamingMessage msg) + { + if (_specializationDone) + { + // Specialization done. So forward all messages to customer payload. + await MessageChannel.Instance.SendInboundAsync(msg); + return; + } + + var responseMessage = new StreamingMessage(); + + switch (msg.ContentCase) + { + case StreamingMessage.ContentOneofCase.WorkerInitRequest: + { + responseMessage.WorkerInitResponse = BuildWorkerInitResponse(); + break; + } + case StreamingMessage.ContentOneofCase.FunctionsMetadataRequest: + { + responseMessage.FunctionMetadataResponse = BuildFunctionMetadataResponse(); + break; + } + case StreamingMessage.ContentOneofCase.FunctionEnvironmentReloadRequest: + + Logger.LogTrace("Specialization request received."); + + var envReloadRequest = msg.FunctionEnvironmentReloadRequest; + foreach (var kv in envReloadRequest.EnvironmentVariables) + { + EnvironmentUtils.SetValue(kv.Key, kv.Value); + } + + var applicationExePath = PathUtils.GetApplicationExePath(envReloadRequest.FunctionAppDirectory); + Logger.LogTrace($"application path {applicationExePath}"); + +#pragma warning disable CS4014 + Task.Run(() => +#pragma warning restore CS4014 + { + _ = _appLoader.RunApplication(applicationExePath); + }); + + Logger.LogTrace($"Will wait for worker loaded signal."); + WorkerLoadStatusSignalManager.Instance.Signal.WaitOne(); + Logger.LogTrace($"Received worker loaded signal. Forwarding environment reload request to worker."); + + await MessageChannel.Instance.SendInboundAsync(msg); + _specializationDone = true; + break; + } + + await MessageChannel.Instance.SendOutboundAsync(responseMessage); + } + + private static FunctionMetadataResponse BuildFunctionMetadataResponse() + { + var metadataResponse = new FunctionMetadataResponse + { + UseDefaultMetadataIndexing = true, + Result = new StatusResult { Status = StatusResult.Types.Status.Success } + }; + + return metadataResponse; + } + + private static WorkerInitResponse BuildWorkerInitResponse() + { + var response = new WorkerInitResponse + { + Result = new StatusResult { Status = StatusResult.Types.Status.Success } + }; + + return response; + } + } +} diff --git a/host/src/FunctionsNetHost/Grpc/MessageChannel.cs b/host/src/FunctionsNetHost/Grpc/MessageChannel.cs new file mode 100644 index 000000000..bf2234c1c --- /dev/null +++ b/host/src/FunctionsNetHost/Grpc/MessageChannel.cs @@ -0,0 +1,61 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Threading.Channels; +using Microsoft.Azure.Functions.Worker.Grpc.Messages; + +namespace FunctionsNetHost.Grpc +{ + /// + /// Bidirectional message channel meant to store inbound(to worker) and outbound(to host) messages. + /// + internal sealed class MessageChannel + { + private MessageChannel() + { + InboundChannel = Channel.CreateUnbounded(CreateUnboundedChannelOptions()); + OutboundChannel = Channel.CreateUnbounded(CreateUnboundedChannelOptions()); + } + + /// + /// Gets the instances of the messaging channel. + /// + internal static MessageChannel Instance { get; } = new(); + + /// + /// Messages which needs to go to worker payload gets pushed to this channel. + /// + internal Channel InboundChannel { get; } + + /// + /// Messages which needs to go to host gets pushed to this channel. + /// + internal Channel OutboundChannel { get; } + + /// + /// Pushes a message to the inbound channel(to worker). + /// + internal async Task SendInboundAsync(StreamingMessage inboundMessage) + { + await InboundChannel.Writer.WriteAsync(inboundMessage); + } + + /// + /// Pushes a messages to the outbound channel(to host) + /// + internal async Task SendOutboundAsync(StreamingMessage outboundMessage) + { + await OutboundChannel.Writer.WriteAsync(outboundMessage); + } + + private static UnboundedChannelOptions CreateUnboundedChannelOptions() + { + return new UnboundedChannelOptions + { + SingleWriter = false, + SingleReader = false, + AllowSynchronousContinuations = true + }; + } + } +} diff --git a/host/src/FunctionsNetHost/Grpc/PathUtils.cs b/host/src/FunctionsNetHost/Grpc/PathUtils.cs new file mode 100644 index 000000000..839d79d0d --- /dev/null +++ b/host/src/FunctionsNetHost/Grpc/PathUtils.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace FunctionsNetHost.Grpc +{ + internal static class PathUtils + { + /// + /// Gets the absolute path to worker application executable. + /// Builds the path by reading the worker.config.json + /// + /// The FunctionAppDirectory value from environment reload request. + internal static string? GetApplicationExePath(string applicationDirectory) + { + string jsonString = string.Empty; + string workerConfigPath = string.Empty; + try + { + workerConfigPath = Path.Combine(applicationDirectory, "worker.config.json"); + + jsonString = File.ReadAllText(workerConfigPath); + var workerConfigJsonNode = JsonNode.Parse(jsonString)!; + var executableName = workerConfigJsonNode["description"]?["defaultWorkerPath"]?.ToString(); + + if (executableName == null) + { + Logger.Log($"Invalid worker configuration. description > defaultWorkerPath property value is null. jsonString:{jsonString}"); + return null; + } + + return Path.Combine(applicationDirectory, executableName); + } + catch (FileNotFoundException ex) + { + Logger.Log($"{workerConfigPath} file not found.{ex}"); + return null; + } + catch (JsonException ex) + { + Logger.Log($"Error parsing JSON in GetApplicationExePath.{ex}. jsonString:{jsonString}"); + return null; + } + catch (Exception ex) + { + Logger.Log($"Error in GetApplicationExePath.{ex}. jsonString:{jsonString}"); + return null; + } + } + } +} diff --git a/host/src/FunctionsNetHost/Logger.cs b/host/src/FunctionsNetHost/Logger.cs new file mode 100644 index 000000000..5731bf7f4 --- /dev/null +++ b/host/src/FunctionsNetHost/Logger.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Globalization; + +namespace FunctionsNetHost +{ + internal static class Logger + { + private static readonly string LogPrefix; + + static Logger() + { +#if !DEBUG + LogPrefix = "LanguageWorkerConsoleLog"; +#else + LogPrefix = ""; +#endif + } + + internal static bool IsTraceLogEnabled + { + get + { + return string.Equals(EnvironmentUtils.GetValue(EnvironmentSettingNames.FunctionsNetHostTrace), "1"); + } + } + + /// + /// Logs a trace message if "AZURE_FUNCTIONS_FUNCTIONSNETHOST_TRACE" environment variable value is set to "1" + /// + internal static void LogTrace(string message) + { + if (IsTraceLogEnabled) + { + Log(message); + } + } + + internal static void Log(string message) + { + var ts = DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture); + Console.WriteLine($"{LogPrefix}[{ts}] [FunctionsNetHost] {message}"); + } + } +} diff --git a/host/src/FunctionsNetHost/Native/NativeExports.cs b/host/src/FunctionsNetHost/Native/NativeExports.cs new file mode 100644 index 000000000..2d44f3015 --- /dev/null +++ b/host/src/FunctionsNetHost/Native/NativeExports.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Runtime.InteropServices; +using FunctionsNetHost.Grpc; +using Microsoft.Azure.Functions.Worker.Grpc.Messages; + +namespace FunctionsNetHost +{ + public static class NativeExports + { + [UnmanagedCallersOnly(EntryPoint = "get_application_properties")] + public static int GetApplicationProperties(NativeHostData nativeHostData) + { + Logger.LogTrace("NativeExports.GetApplicationProperties method invoked."); + + try + { + var nativeHostApplication = NativeHostApplication.Instance; + GCHandle gch = GCHandle.Alloc(nativeHostApplication, GCHandleType.Pinned); + IntPtr pNativeApplication = gch.AddrOfPinnedObject(); + nativeHostData.PNativeApplication = pNativeApplication; + + return 1; + } + catch (Exception ex) + { + Logger.Log($"Error in NativeExports.GetApplicationProperties: {ex}"); + return 0; + } + } + + [UnmanagedCallersOnly(EntryPoint = "register_callbacks")] + public static unsafe int RegisterCallbacks(IntPtr pInProcessApplication, + delegate* unmanaged requestCallback, + IntPtr grpcHandler) + { + Logger.LogTrace("NativeExports.RegisterCallbacks method invoked."); + + try + { + NativeHostApplication.Instance.SetCallbackHandles(requestCallback, grpcHandler); + return 1; + } + catch (Exception ex) + { + Logger.Log($"Error in RegisterCallbacks: {ex}"); + return 0; + } + } + + [UnmanagedCallersOnly(EntryPoint = "send_streaming_message")] + public static unsafe int SendStreamingMessage(IntPtr pInProcessApplication, byte* streamingMessage, int streamingMessageSize) + { + try + { + var span = new ReadOnlySpan(streamingMessage, streamingMessageSize); + var outboundMessageToHost = StreamingMessage.Parser.ParseFrom(span); + + _ = MessageChannel.Instance.SendOutboundAsync(outboundMessageToHost); + + return 1; + } + catch (Exception ex) + { + Logger.Log($"Error in SendStreamingMessage: {ex}"); + return 0; + } + } + } +} diff --git a/host/src/FunctionsNetHost/Native/NativeHostApplication.cs b/host/src/FunctionsNetHost/Native/NativeHostApplication.cs new file mode 100644 index 000000000..8bcee10ed --- /dev/null +++ b/host/src/FunctionsNetHost/Native/NativeHostApplication.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace FunctionsNetHost +{ + public sealed class NativeHostApplication + { + private IntPtr _workerHandle; + unsafe delegate* unmanaged _requestHandlerCallback; + public static NativeHostApplication Instance { get; } = new(); + + private NativeHostApplication() + { + } + + public unsafe void HandleInboundMessage(byte[] buffer, int size) + { + fixed (byte* pBuffer = buffer) + { + _requestHandlerCallback(&pBuffer, size, _workerHandle); + } + } + + public unsafe void SetCallbackHandles(delegate* unmanaged callback, IntPtr grpcHandle) + { + _requestHandlerCallback = callback; + _workerHandle = grpcHandle; + + WorkerLoadStatusSignalManager.Instance.Signal.Set(); + } + } +} diff --git a/host/src/FunctionsNetHost/Native/NativeHostData.cs b/host/src/FunctionsNetHost/Native/NativeHostData.cs new file mode 100644 index 000000000..07162c18a --- /dev/null +++ b/host/src/FunctionsNetHost/Native/NativeHostData.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace FunctionsNetHost; + +public struct NativeHostData +{ + public IntPtr PNativeApplication; +} diff --git a/host/src/FunctionsNetHost/Program.cs b/host/src/FunctionsNetHost/Program.cs new file mode 100644 index 000000000..3f8def5eb --- /dev/null +++ b/host/src/FunctionsNetHost/Program.cs @@ -0,0 +1,66 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.CommandLine; +using FunctionsNetHost.Grpc; + +namespace FunctionsNetHost +{ + internal class Program + { + static async Task Main(string[] args) + { + try + { + Logger.Log("Starting FunctionsNetHost"); + + var workerStartupOptions = await GetStartupOptionsFromCmdLineArgs(args); + + + using var appLoader = new AppLoader(); + var grpcClient = new GrpcClient(workerStartupOptions, appLoader); + + await grpcClient.InitAsync(); + } + catch (Exception exception) + { + Logger.Log($"An error occurred while running FunctionsNetHost.{exception}"); + } + } + + private static async Task GetStartupOptionsFromCmdLineArgs(string[] args) + { + var hostOption = new Option("--host"); + var portOption = new Option("--port"); + var workerOption = new Option("--workerId"); + var grpcMsgLengthOption = new Option("--grpcMaxMessageLength"); + var requestIdOption = new Option("--requestId"); + + var rootCommand = new RootCommand(); + rootCommand.AddOption(portOption); + rootCommand.AddOption(hostOption); + rootCommand.AddOption(workerOption); + rootCommand.AddOption(grpcMsgLengthOption); + rootCommand.AddOption(requestIdOption); + + var workerStartupOptions = new GrpcWorkerStartupOptions(); + + rootCommand.SetHandler((host, port, workerId, grpcMsgLength, requestId) => + { + workerStartupOptions.Host = host; + workerStartupOptions.Port = port; + workerStartupOptions.WorkerId = workerId; + workerStartupOptions.GrpcMaxMessageLength = grpcMsgLength; + workerStartupOptions.RequestId = requestId; + }, + hostOption, portOption, workerOption, grpcMsgLengthOption, requestIdOption); + + Logger.LogTrace($"raw args:{string.Join(" ", args)}"); + + var argsWithoutExecutableName = args.Skip(1).ToArray(); + await rootCommand.InvokeAsync(argsWithoutExecutableName); + + return workerStartupOptions; + } + } +} diff --git a/host/src/FunctionsNetHost/Properties/launchSettings.json b/host/src/FunctionsNetHost/Properties/launchSettings.json new file mode 100644 index 000000000..c6b3919b9 --- /dev/null +++ b/host/src/FunctionsNetHost/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "FunctionsNetHost": { + "commandName": "Project", + "commandLineArgs": "FunctionsNetHost.exe --host 127.0.0.1 --port 503 --workerId 13a2c943-ee61-449b-97ea-7c2577cbb1db --requestId 78522dbc-3bef-4ced-8988-bb3761c94e00 --grpcMaxMessageLength 2147483647" + } + } +} \ No newline at end of file diff --git a/host/src/FunctionsNetHost/exports.def b/host/src/FunctionsNetHost/exports.def new file mode 100644 index 000000000..b350d24ca --- /dev/null +++ b/host/src/FunctionsNetHost/exports.def @@ -0,0 +1,4 @@ +EXPORTS + get_application_properties + register_callbacks + send_streaming_message diff --git a/host/src/FunctionsNetHost/global.json b/host/src/FunctionsNetHost/global.json new file mode 100644 index 000000000..e90889403 --- /dev/null +++ b/host/src/FunctionsNetHost/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "8.0.100-preview.6.23330.14", + "allowPrerelease": true, + "rollForward": "latestMinor" + } +} \ No newline at end of file diff --git a/host/src/FunctionsNetHost/main.cpp b/host/src/FunctionsNetHost/main.cpp deleted file mode 100644 index a0d22d109..000000000 --- a/host/src/FunctionsNetHost/main.cpp +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#include "../funcgrpc/func_bidi_reactor.h" -#include "../funcgrpc/func_perf_marker.h" -#include -#include -#include -#include -#include - -using namespace std; -using namespace funcgrpc; -namespace po = boost::program_options; - -unique_ptr getWorkerStartupOptions(int argc, char *const *argv); - -int main(int argc, char *argv[]) -{ - funcgrpc::Log::Init(); - FUNC_LOG_INFO("Starting FunctionsNetHost."); - - try - { - auto pOptions = getWorkerStartupOptions(argc, argv); - auto pApplication = std::make_unique(); - auto worker = std::make_unique(pOptions.get(), pApplication.get()); - Status status = worker->Await(); - - if (!status.ok()) - { - FUNC_LOG_ERROR("Rpc failed. error_message:{}", status.error_message()); - } - } - catch (const std::exception &ex) - { - FUNC_LOG_ERROR("Caught unknown exception.{}", ex.what()); - } - catch (...) - { - FUNC_LOG_ERROR("Caught unknown exception."); - } - - return 0; -} - -unique_ptr getWorkerStartupOptions(int argc, char *const *argv) -{ - FuncPerfMarker marker("BuildWorkerStartupOptions"); - - po::options_description desc("Allowed options"); - desc.add_options()("help", "sample usage: FunctionsNetHost --host --port --workerid " - "--requestid --grpcmaxrequestlength ")( - "host", boost::program_options::value(), - "Address of grpc server")("port", po::value(), "Port number of grpc server connection")( - "workerId", boost::program_options::value(), - "Worker id")("requestId", boost::program_options::value(), - "Request id")("grpcMaxMessageLength", po::value()->default_value(INT_MAX), - "Max length for grpc messages. Default is INT_MAX"); - - po::variables_map vm; - po::store(po::parse_command_line(argc, argv, desc), vm); - po::notify(vm); - - auto options = make_unique(); - - if (vm.count("host")) - { - options->host = vm["host"].as(); - } - if (vm.count("port")) - { - options->port = vm["port"].as(); - } - if (vm.count("workerId")) - { - options->workerId = vm["workerId"].as(); - } - if (vm.count("requestId")) - { - options->requestId = vm["requestId"].as(); - } - if (vm.count("grpcMaxMessageLength")) - { - options->grpcMaxMessageLength = vm["grpcMaxMessageLength"].as(); - } - - return options; -} diff --git a/host/src/FunctionsNetHost/managedexports.cpp b/host/src/FunctionsNetHost/managedexports.cpp deleted file mode 100644 index 457fab678..000000000 --- a/host/src/FunctionsNetHost/managedexports.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#include "../funcgrpc/byte_buffer_helper.h" -#include "../funcgrpc/func_log.h" -#include "../funcgrpc/nativehostapplication.h" -#include -#include -struct NativeHostData -{ - NativeHostApplication *pNativeApplication; -}; - -extern "C" __declspec(dllexport) HRESULT get_application_properties(_In_ NativeHostData *pNativeHostData) -{ - auto pInProcessApplication = NativeHostApplication::GetInstance(); - - if (pInProcessApplication == nullptr) - { - return E_FAIL; - } - - pNativeHostData->pNativeApplication = pInProcessApplication; - - return S_OK; -} - -extern "C" __declspec(dllexport) HRESULT send_streaming_message(_In_ NativeHostApplication *pInProcessApplication, - _In_ char *managedMessage, _In_ int managedMessageSize) -{ - FUNC_LOG_DEBUG("Calling send_streaming_message. managedMessageSize:{}", managedMessageSize); - - if (managedMessageSize == 0) - { - FUNC_LOG_WARN("send_streaming_message. size 0"); - return S_OK; - } - - auto bbUPtr = funcgrpc::SerializeToByteBufferFromChar(managedMessage, managedMessageSize); - auto byteBuffer = bbUPtr.get(); - pInProcessApplication->SendOutgoingMessage(byteBuffer); - - return S_OK; -} - -extern "C" __declspec(dllexport) HRESULT - register_callbacks(_In_ NativeHostApplication *pInProcessApplication, _In_ PFN_REQUEST_HANDLER request_handler, - _In_ VOID *grpcHandler) -{ - if (pInProcessApplication == nullptr) - { - return E_INVALIDARG; - } - - pInProcessApplication->SetCallbackHandles(request_handler, grpcHandler); - - return S_OK; -} \ No newline at end of file diff --git a/host/src/Protos/CMakeLists.txt b/host/src/Protos/CMakeLists.txt deleted file mode 100644 index e118f2806..000000000 --- a/host/src/Protos/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -find_package(protobuf CONFIG REQUIRED) -find_package(gRPC CONFIG REQUIRED) -find_package(Threads) - -# We cannot directly refer the proto files in the repo root as that dir is not a CMAKE Source dir. -# So we will copy protos files to a relative location. -get_filename_component(HOST_DIR ${CMAKE_SOURCE_DIR} DIRECTORY) -get_filename_component(REPO_ROOT ${HOST_DIR} DIRECTORY) -SET (PROTOS_TARGET_DIR ${CMAKE_SOURCE_DIR}/Protos) -SET (PROTOS_SRC_DIR ${REPO_ROOT}/protos/azure-functions-language-worker-protobuf/src/proto) - -# Copy proto files to the "Protos" directory(where this CMakeList.txt file is present). -file(COPY ${PROTOS_SRC_DIR}/shared/NullableTypes.proto DESTINATION ${PROTOS_TARGET_DIR}/shared) -file(COPY ${PROTOS_SRC_DIR}/identity/ClaimsIdentityRpc.proto DESTINATION ${PROTOS_TARGET_DIR}/identity) -file(COPY ${PROTOS_SRC_DIR}/FunctionRpc.proto DESTINATION ${PROTOS_TARGET_DIR}) - -# -# Protobuf/Grpc source files for function RPC -# -set(PROTO_FILES - FunctionRpc.proto - shared/NullableTypes.proto - identity/ClaimsIdentityRpc.proto -) - -# -# Add Library target with protobuf sources -# -add_library(func_protos ${PROTO_FILES}) -target_link_libraries(func_protos - PUBLIC - protobuf::libprotobuf - gRPC::grpc - gRPC::grpc++ -) - -target_include_directories(func_protos PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) - -# -# Compile protobuf and grpc files -# -get_target_property(grpc_cpp_plugin_location gRPC::grpc_cpp_plugin LOCATION) -protobuf_generate(TARGET func_protos LANGUAGE cpp) -protobuf_generate(TARGET func_protos LANGUAGE grpc GENERATE_EXTENSIONS .grpc.pb.h .grpc.pb.cc PLUGIN "protoc-gen-grpc=${grpc_cpp_plugin_location}") \ No newline at end of file diff --git a/host/src/README.md b/host/src/README.md index 03d672106..e9ce2556d 100644 --- a/host/src/README.md +++ b/host/src/README.md @@ -1,11 +1,7 @@ +# FunctionsNetHost -This is the project root where we have our root level CMakeLists.txt. -### Load & Build. +Managed code version of FunctionsNetHost. -Open this directory in Visual studio(Open a local folder). -VS will read the CMakeLists.txt and start executing CMake. -We use [vcpkg](https://vcpkg.io) for dependency management. When CMake execution starts, it will start downloading the dependencies to the build output directory.This may take a while to finish the first time. +## Publish -Dependencies are listed in the vcpkg.json file. - -Once CMake generation is done, build the code by Build-> Build All (or F6 key). It will do compilation and linking. +Open a terminal here and run `dotnet publish -c release -r win-x64`. This will produce the native exe in `FunctionsNetHost\bin\Release\net7.0\win-x64\publish\` directory. diff --git a/host/src/funcgrpc/CMakeLists.txt b/host/src/funcgrpc/CMakeLists.txt deleted file mode 100644 index d11dfd336..000000000 --- a/host/src/funcgrpc/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -# Add source to this project's executable. -add_library(funcgrpc - "funcgrpc_handlers.h" - "funcgrpc.h" - "func_bidi_reactor.h" - "func_bidi_reactor.cpp" - "handlers/funcgrpc_native_handler.h" - "handlers/funcgrpc_native_handler.cpp" - "messaging_channel.h" - "messaging_channel.cpp" - "nativehostapplication.cpp" - "nativehostapplication.h" - func_perf_marker.h - func_log.cpp - func_log.h - funcgrpc_worker_config_handle.cpp - funcgrpc_worker_config_handle.h - byte_buffer_helper.cpp - byte_buffer_helper.h) - -# set_target_properties(funcgrpc PROPERTIES ENABLE_EXPORTS TRUE) -# set_target_properties(funcgrpc PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) - -target_link_libraries(funcgrpc PUBLIC func_protos) - -find_package(Boost REQUIRED COMPONENTS fiber) -target_link_libraries(funcgrpc PUBLIC Boost::fiber) - -#find_package(Boost REQUIRED COMPONENTS json) -#target_link_libraries(funcgrpc PUBLIC Boost::json) - -#find_package(nlohmann_json 3.2.0 REQUIRED) -#target_link_libraries(funcgrpc PRIVATE nlohmann_json::nlohmann_json) - -find_package(unofficial-nethost CONFIG REQUIRED) -target_link_libraries(funcgrpc PRIVATE unofficial::nethost::nethost) - -if (NOT TARGET spdlog) - find_package(spdlog REQUIRED) -endif () -target_link_libraries(funcgrpc PRIVATE spdlog::spdlog) \ No newline at end of file diff --git a/host/src/funcgrpc/byte_buffer_helper.cpp b/host/src/funcgrpc/byte_buffer_helper.cpp deleted file mode 100644 index dfa6add15..000000000 --- a/host/src/funcgrpc/byte_buffer_helper.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "byte_buffer_helper.h" -#include - -// these bytebuffer helpers were copied from e2e tests of https://github.com/grpc/grpc - -std::unique_ptr funcgrpc::SerializeToByteBuffer(protobuf::Message *message) -{ - grpc::string buf; - message->SerializePartialToString(&buf); - Slice slice(buf); - - return std::make_unique(&slice, 1); -} - -bool funcgrpc::ParseFromByteBuffer(ByteBuffer *buffer, protobuf::Message *message) -{ - std::vector slices; - (void)buffer->Dump(&slices); - grpc::string buf; - buf.reserve(buffer->Length()); - for (auto s = slices.begin(); s != slices.end(); s++) - { - buf.append(reinterpret_cast(s->begin()), s->size()); - } - - return message->ParseFromString(buf); -} - -std::unique_ptr funcgrpc::SerializeToByteBufferFromChar(char *managedMessage, int managedMessageSize) -{ - grpc::string buf(managedMessage, managedMessageSize); - Slice slice(buf); - - return std::make_unique(&slice, 1); -} - -string funcgrpc::ParseFromByteBufferToString(ByteBuffer *buffer) -{ - std::vector slices; - (void)buffer->Dump(&slices); - grpc::string buf; - buf.reserve(buffer->Length()); - for (auto s = slices.begin(); s != slices.end(); s++) - { - buf.append(reinterpret_cast(s->begin()), s->size()); - } - - return buf; -} diff --git a/host/src/funcgrpc/byte_buffer_helper.h b/host/src/funcgrpc/byte_buffer_helper.h deleted file mode 100644 index b14c74f2f..000000000 --- a/host/src/funcgrpc/byte_buffer_helper.h +++ /dev/null @@ -1,23 +0,0 @@ - -#ifndef FUNCTIONSNETHOST_BYTE_BUFFER_HELPER_H -#define FUNCTIONSNETHOST_BYTE_BUFFER_HELPER_H - -#include "grpcpp/impl/codegen/config_protobuf.h" -#include "grpcpp/support/byte_buffer.h" -#include "iostream" -#include -#include - -using namespace std; -using namespace grpc; -namespace funcgrpc -{ -std::unique_ptr SerializeToByteBuffer(grpc::protobuf::Message *message); - -bool ParseFromByteBuffer(grpc::ByteBuffer *buffer, grpc::protobuf::Message *message); - -string ParseFromByteBufferToString(ByteBuffer *buffer); - -std::unique_ptr SerializeToByteBufferFromChar(char *managedMessage, int managedMessageSize); -} -#endif diff --git a/host/src/funcgrpc/func_bidi_reactor.cpp b/host/src/funcgrpc/func_bidi_reactor.cpp deleted file mode 100644 index 9c272ecab..000000000 --- a/host/src/funcgrpc/func_bidi_reactor.cpp +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#include "func_bidi_reactor.h" -#include "byte_buffer_helper.h" -#include "func_log.h" -#include "handlers/funcgrpc_native_handler.h" -#include "messaging_channel.h" - -funcgrpc::FunctionBidiReactor::FunctionBidiReactor(GrpcWorkerStartupOptions *pOptions, - NativeHostApplication *pApplication) -{ - pOptions_ = pOptions; - pApplication_ = pApplication; - - std::string endpoint = pOptions->host + ":" + std::to_string(pOptions->port); - grpc::ChannelArguments channelArgs; - channelArgs.SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, pOptions->grpcMaxMessageLength); - channelArgs.SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, pOptions->grpcMaxMessageLength); - auto channel = grpc::CreateCustomChannel(endpoint, grpc::InsecureChannelCredentials(), channelArgs); - - auto generic_stub_ = make_unique(channel); - const char *suffix_for_stats = nullptr; - grpc::StubOptions options(suffix_for_stats); - generic_stub_->PrepareBidiStreamingCall(&client_context_, "/AzureFunctionsRpcMessages.FunctionRpc/EventStream", - options, this); - handler_ = std::unique_ptr(new NativeHostMessageHandler(pApplication)); - - sendStartStream(); - StartRead(&read_); - StartCall(); - - auto outboundWriterTask = std::async(std::launch::async, [this]() { startOutboundWriter(); }); - auto inboundMsgHandlingTask = std::async(std::launch::async, [this]() { handleInboundMessagesForApplication(); }); -} - -void funcgrpc::FunctionBidiReactor::OnWriteDone(bool ok) -{ - FUNC_LOG_TRACE("OnWriteDone. ok:{}", ok); - - { - bool expect = true; - if (!write_inprogress_.compare_exchange_strong(expect, false, std::memory_order_relaxed)) - { - FUNC_LOG_WARN("Illegal write_inprogress_ state"); - } - } - - fireWrite(); -} - -void funcgrpc::FunctionBidiReactor::OnReadDone(bool ok) -{ - if (!ok) - { - FUNC_LOG_WARN("Failed to read response."); - return; - } - - grpc::ByteBuffer outboundMessage(read_); - - auto handleMsgTask = std::async(std::launch::async, [this, &outboundMessage]() { - auto outboundStreamingMsg = StreamingMessage(); - handler_->HandleMessage(&outboundMessage); - }); - - fireRead(); -} - -void funcgrpc::FunctionBidiReactor::OnDone(const grpc::Status &status) -{ - if (status.ok()) - { - FUNC_LOG_DEBUG("Bi-directional stream ended. status.code={}, status.message={}", status.error_code(), - status.error_message()); - } - - std::unique_lock l(mu_); - status_ = status; - done_ = true; - cv_.notify_one(); -} - -Status funcgrpc::FunctionBidiReactor::Await() -{ - std::unique_lock l(mu_); - cv_.wait(l, [this] { return done_; }); - return std::move(status_); -} - -/// -/// Manages outbound(to host) write operations. -/// Listening to the outbound channel and when a new message arrives, -/// we will push that entry to the write buffer. -/// - -void funcgrpc::FunctionBidiReactor::startOutboundWriter() -{ - FUNC_LOG_DEBUG("startOutboundWriter started"); - auto &outboundChannel = funcgrpc::MessageChannel::GetInstance().GetOutboundChannel(); - - grpc::ByteBuffer messagetoSend; - while (channel_pop_status_t::success == outboundChannel.pop(messagetoSend)) - { - FUNC_LOG_DEBUG("Popped new message received in outbound channel"); - writeToOutboundBuffer(messagetoSend); - } - FUNC_LOG_WARN("exiting startOutboundWriter."); -} - -/// -/// Sends the startStream message to host. -/// This will initiate the GRPC communication with host. -/// -void funcgrpc::FunctionBidiReactor::sendStartStream() -{ - StreamingMessage startStream; - startStream.mutable_start_stream()->set_worker_id(pOptions_->workerId); - FUNC_LOG_INFO("Sending StartStream message."); - - auto bbUniqPtr = funcgrpc::SerializeToByteBuffer(&startStream); - auto bbPtr = bbUniqPtr.get(); - writeToOutboundBuffer(*bbPtr); -} - -/// -/// Pushes an outgoing message(to host) to the buffer. -/// -void funcgrpc::FunctionBidiReactor::writeToOutboundBuffer(const grpc::ByteBuffer &outgoingMessage) -{ - { - absl::MutexLock lk(&writes_mtx_); - writes_.push_back(outgoingMessage); - FUNC_LOG_TRACE("Pushed entry to writes_ buffer"); - } - fireWrite(); -} - -/// -/// Pull the message from buffer and writes to GRPC outbound stream. -/// -void funcgrpc::FunctionBidiReactor::fireWrite() -{ - { - absl::MutexLock lk(&writes_mtx_); - if (writes_.empty()) - { - return; - } - - bool expect = false; - if (write_inprogress_.compare_exchange_strong(expect, true, std::memory_order_relaxed)) - { - write_ = *writes_.begin(); - writes_.erase(writes_.begin()); - } - else - { - FUNC_LOG_DEBUG("Another write operation is in progress."); - return; - } - } - - StartWrite(&write_); -} - -void funcgrpc::FunctionBidiReactor::fireRead() -{ - FUNC_LOG_TRACE("fireRead called"); - StartRead(&read_); -} - -/// -/// Handles messages meant for the application/dotnet worker. -/// Listening to the inbound channel and when a new message arrives, -/// we will send that to the application_. -/// -/// TO DO: I think we can move this to the funcgrpc_native_handler.cpp. -/// -/// -void funcgrpc::FunctionBidiReactor::handleInboundMessagesForApplication() -{ - FUNC_LOG_DEBUG("handleInboundMessagesForApplication started"); - - auto &inboundChannel = funcgrpc::MessageChannel::GetInstance().GetInboundChannel(); - grpc::ByteBuffer ibByteBuffer; - - while (channel_pop_status_t::success == inboundChannel.pop(ibByteBuffer)) - { - FUNC_LOG_DEBUG("Popped new message received in inbound channel"); - - auto size = ibByteBuffer.Length(); - std::string t = funcgrpc::ParseFromByteBufferToString(&ibByteBuffer); - auto charArr = t.c_str(); - auto *unsignedCharArr = (unsigned char *)charArr; - - pApplication_->HandleIncomingMessage(unsignedCharArr, size); - } - - FUNC_LOG_WARN("exiting handleInboundMessagesForApplication"); -} \ No newline at end of file diff --git a/host/src/funcgrpc/func_bidi_reactor.h b/host/src/funcgrpc/func_bidi_reactor.h deleted file mode 100644 index d153cade7..000000000 --- a/host/src/funcgrpc/func_bidi_reactor.h +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "funcgrpc.h" -#include "funcgrpc_handlers.h" -#include "nativehostapplication.h" -#include -#include - -using AzureFunctionsRpcMessages::FunctionLoadResponse; -using AzureFunctionsRpcMessages::FunctionRpc; -using AzureFunctionsRpcMessages::StartStream; -using AzureFunctionsRpcMessages::StatusResult; -using AzureFunctionsRpcMessages::StreamingMessage; -using AzureFunctionsRpcMessages::WorkerInitResponse; -using grpc::ByteBuffer; -using grpc::Channel; -using grpc::ClientContext; -using grpc::Status; - -using namespace AzureFunctionsRpc; -using namespace grpc; -namespace funcgrpc -{ - -/// -/// BidiReactor implementation which reads and writes messages from the GRPC stream asynchronously. -/// See https://github.com/grpc/proposal/blob/master/L67-cpp-callback-api.md for details. -/// -class FunctionBidiReactor : public grpc::ClientBidiReactor -{ - public: - FunctionBidiReactor(GrpcWorkerStartupOptions *options, NativeHostApplication *application); - - void OnWriteDone(bool ok) override; - - void OnReadDone(bool ok) override; - - void OnDone(const Status &status) override; - - Status Await(); - - void startOutboundWriter(); - - void sendStartStream(); - - void handleInboundMessagesForApplication(); - - /// - /// Push a new message to the buffer. - /// - /// - void writeToOutboundBuffer(const ByteBuffer &outgoingMessage); - - /// - /// Writes the next message in buffer to the outbound GRPC stream. - /// - void fireWrite(); - - /// - /// Reads from GRPC stream. - /// - void fireRead(); - - private: - GrpcWorkerStartupOptions *pOptions_; - std::unique_ptr handler_; - NativeHostApplication *pApplication_; - - std::mutex mu_; - std::condition_variable cv_; - Status status_; - bool done_ = false; - - grpc::ClientContext client_context_; - - /// - /// Message to read from server. - /// This should not be modified while a read operation( StartRead(&read_) ) is in progress. - /// - ByteBuffer read_; - - /// - /// Message to write to server. - /// This should not be modified while a write operation( StartWrite(&write_) ) is in progress. - /// - ByteBuffer write_; - - std::atomic_bool write_inprogress_{false}; - - // Buffer for writing operations. - std::vector writes_ GUARDED_BY(writes_mtx_); - absl::Mutex writes_mtx_; -}; -} // namespace funcgrpc \ No newline at end of file diff --git a/host/src/funcgrpc/func_log.cpp b/host/src/funcgrpc/func_log.cpp deleted file mode 100644 index 683cb5390..000000000 --- a/host/src/funcgrpc/func_log.cpp +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#include "func_log.h" - -namespace funcgrpc -{ -std::shared_ptr Log::funcLogger; - -void Log::Init() -{ - funcLogger = spdlog::stdout_color_mt("FunctionsNetHost"); - spdlog::set_pattern("LanguageWorkerConsoleLog%^[%H:%M:%S.%e] [%l] %n: %v%$"); - funcLogger->set_level(spdlog::level::info); - -#if defined(_DEBUG) || defined(DEBUG) - funcLogger->flush_on(spdlog::level::trace); -#elif defined(NDEBUG) - funcLogger->flush_on(spdlog::level::warn); -#endif -} -} // namespace funcgrpc \ No newline at end of file diff --git a/host/src/funcgrpc/func_log.h b/host/src/funcgrpc/func_log.h deleted file mode 100644 index 366b7e25e..000000000 --- a/host/src/funcgrpc/func_log.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#pragma once - -#include "spdlog/sinks/stdout_color_sinks.h" -#include "spdlog/spdlog.h" -#include -#include - -namespace funcgrpc -{ -class Log -{ - public: - static void Init(); - - private: - static std::shared_ptr funcLogger; -}; -} // namespace funcgrpc - -#if defined(_DEBUG) || defined(DEBUG) -#define FUNC_LOG_TRACE(...) spdlog::get("FunctionsNetHost")->trace(__VA_ARGS__) -#define FUNC_LOG_DEBUG(...) spdlog::get("FunctionsNetHost")->debug(__VA_ARGS__) -#define FUNC_LOG_INFO(...) spdlog::get("FunctionsNetHost")->info(__VA_ARGS__) -#define FUNC_LOG_WARN(...) spdlog::get("FunctionsNetHost")->warn(__VA_ARGS__) -#define FUNC_LOG_ERROR(...) spdlog::get("FunctionsNetHost")->error(__VA_ARGS__) -#elif defined(NDEBUG) -#define FUNC_LOG_TRACE(...) (void)0 -#define FUNC_LOG_DEBUG(...) (void)0 -#define FUNC_LOG_INFO(...) spdlog::get("FunctionsNetHost")->info(__VA_ARGS__) -#define FUNC_LOG_WARN(...) spdlog::get("FunctionsNetHost")->warn(__VA_ARGS__) -#define FUNC_LOG_ERROR(...) spdlog::get("FunctionsNetHost")->error(__VA_ARGS__) -#endif \ No newline at end of file diff --git a/host/src/funcgrpc/func_perf_marker.h b/host/src/funcgrpc/func_perf_marker.h deleted file mode 100644 index 92abc4f4e..000000000 --- a/host/src/funcgrpc/func_perf_marker.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#ifndef FUNCTIONSNETHOST_FUNC_PERF_MARKER_H -#define FUNCTIONSNETHOST_FUNC_PERF_MARKER_H - -#include "func_log.h" -#include -#include -#include -#include -namespace funcgrpc -{ -class FuncPerfMarker -{ - public: - explicit FuncPerfMarker(const std::string &name) - { - _name = name; - _start = std::chrono::high_resolution_clock::now(); - } - - ~FuncPerfMarker() - { - auto stop = std::chrono::high_resolution_clock::now(); - auto durationMs = duration_cast(stop - _start); - FUNC_LOG_INFO("{} elapsed: {}ms", _name, durationMs.count()); - } - - private: - std::chrono::time_point _start; - std::string _name; -}; -} // namespace funcgrpc -#endif diff --git a/host/src/funcgrpc/funcgrpc.h b/host/src/funcgrpc/funcgrpc.h deleted file mode 100644 index affe1281a..000000000 --- a/host/src/funcgrpc/funcgrpc.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#ifndef FUNC_WORKER -#define FUNC_WORKER - -#include "funcgrpc_handlers.h" -#include -#include - -using namespace AzureFunctionsRpc; - -namespace funcgrpc -{ - -class GrpcWorkerStartupOptions -{ - public: - GrpcWorkerStartupOptions() = default; - ; - std::string host; - int port; - std::string workerId; - std::string requestId; - int grpcMaxMessageLength; -}; -} // namespace funcgrpc - -#endif \ No newline at end of file diff --git a/host/src/funcgrpc/funcgrpc_handlers.h b/host/src/funcgrpc/funcgrpc_handlers.h deleted file mode 100644 index b7b463bf7..000000000 --- a/host/src/funcgrpc/funcgrpc_handlers.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#ifndef FUNC_HANDLERS -#define FUNC_HANDLERS - -#include -#include -#include -#include -#include -#include -#include - -using AzureFunctionsRpcMessages::StreamingMessage; -using grpc::ByteBuffer; -namespace AzureFunctionsRpc -{ - -class MessageHandler -{ - public: - virtual void HandleMessage(ByteBuffer *receivedMessage){}; -}; - -} // namespace AzureFunctionsRpc - -#endif \ No newline at end of file diff --git a/host/src/funcgrpc/funcgrpc_worker_config_handle.cpp b/host/src/funcgrpc/funcgrpc_worker_config_handle.cpp deleted file mode 100644 index 4124eccc8..000000000 --- a/host/src/funcgrpc/funcgrpc_worker_config_handle.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#include "funcgrpc_worker_config_handle.h" -#include "func_log.h" -#include "func_perf_marker.h" -#include "rapidjson/document.h" -#include "rapidjson/filereadstream.h" -std::string funcgrpc::WorkerConfigHandle::GetApplicationExePath(const std::string &dir) -{ - std::string fullPath; - try - { - funcgrpc::FuncPerfMarker mark("WorkerConfigHandle->GetApplicationExePath"); - - fullPath = dir + "/worker.config.json"; - - fp = fopen(fullPath.c_str(), "r"); - char readBuffer[65536]; - rapidjson::FileReadStream is(fp, readBuffer, sizeof(readBuffer)); - - rapidjson::Document doc; - doc.ParseStream(is); - closeFileHandle(); - - if (doc.HasParseError()) - { - FUNC_LOG_ERROR("Error parsing {} to rapidJson::Document.", fullPath); - } - std::string workerPath = doc["description"]["defaultWorkerPath"].GetString(); - std::string exePath = dir + "/" + workerPath; - - return exePath; - } - catch (std::exception &ex) - { - FUNC_LOG_ERROR("Error parsing {} to rapidJson::Document.", fullPath); - throw; - } -} -funcgrpc::WorkerConfigHandle::~WorkerConfigHandle() -{ - closeFileHandle(); -} -void funcgrpc::WorkerConfigHandle::closeFileHandle() -{ - if (fp != nullptr) - { - fclose(fp); - fp = nullptr; - } -} diff --git a/host/src/funcgrpc/funcgrpc_worker_config_handle.h b/host/src/funcgrpc/funcgrpc_worker_config_handle.h deleted file mode 100644 index cba86abbf..000000000 --- a/host/src/funcgrpc/funcgrpc_worker_config_handle.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#pragma once - -#include - -namespace funcgrpc -{ -class WorkerConfigHandle -{ - public: - WorkerConfigHandle() = default; - - ~WorkerConfigHandle(); - - /** - * Gets the full path to the function app executable. - * @param dir Path to function app directory. - */ - std::string GetApplicationExePath(const std::string &dir); - - private: - FILE *fp; - void closeFileHandle(); -}; -} // namespace funcgrpc diff --git a/host/src/funcgrpc/handlers/funcgrpc_native_handler.cpp b/host/src/funcgrpc/handlers/funcgrpc_native_handler.cpp deleted file mode 100644 index 8406c651d..000000000 --- a/host/src/funcgrpc/handlers/funcgrpc_native_handler.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#include "funcgrpc_native_handler.h" -#include "../byte_buffer_helper.h" -#include "../func_log.h" -#include "../func_perf_marker.h" -#include "../funcgrpc_worker_config_handle.h" - -using namespace AzureFunctionsRpc; -using AzureFunctionsRpcMessages::FunctionEnvironmentReloadResponse; -using AzureFunctionsRpcMessages::FunctionLoadResponse; -using AzureFunctionsRpcMessages::FunctionMetadataResponse; -using AzureFunctionsRpcMessages::StartStream; -using AzureFunctionsRpcMessages::StatusResult; -using AzureFunctionsRpcMessages::WorkerInitResponse; - -using namespace std; - -AzureFunctionsRpc::NativeHostMessageHandler::NativeHostMessageHandler(NativeHostApplication *application) - : MessageHandler() -{ - application_ = application; -} - -void AzureFunctionsRpc::NativeHostMessageHandler::HandleMessage(ByteBuffer *receivedMessageBb) -{ - if (specializationRequestReceived) - { - // Once we received specialization request & returned a response for that, - // We do not need to deserialize the byte buffer version of message. - FUNC_LOG_DEBUG("New message received in handler. Pushing to InboundChannel."); - - // Forward to inbound channel(managed code wrapper is listening to that channel). - funcgrpc::MessageChannel::GetInstance().GetInboundChannel().push(*receivedMessageBb); - } - else - { - // We will deserialize the bytebuffer version as we need some property values. - StreamingMessage receivedMessage; - funcgrpc::ParseFromByteBuffer(receivedMessageBb, &receivedMessage); - StreamingMessage::ContentCase contentCase = receivedMessage.content_case(); - FUNC_LOG_DEBUG("New message received. contentCase: {}", contentCase); - - if (contentCase == StreamingMessage::ContentCase::kWorkerInitRequest) - { - StreamingMessage streamingMsg; - streamingMsg.mutable_worker_init_response()->mutable_result()->set_status( - AzureFunctionsRpcMessages::StatusResult::Success); - streamingMsg.mutable_worker_init_response()->set_worker_version("1.0.0.2"); - auto uPtrBb = funcgrpc::SerializeToByteBuffer(&streamingMsg); - auto byteBuffer = uPtrBb.get(); - - FUNC_LOG_DEBUG("Pushing response to OutboundChannel.contentCase: {}", streamingMsg.content_case()); - funcgrpc::MessageChannel::GetInstance().GetOutboundChannel().push(*byteBuffer); - } - else if (contentCase == StreamingMessage::ContentCase::kFunctionsMetadataRequest) - { - StreamingMessage streamingMsg; - streamingMsg.mutable_function_metadata_response()->mutable_result()->set_status( - AzureFunctionsRpcMessages::StatusResult::Success); - streamingMsg.mutable_function_metadata_response()->set_use_default_metadata_indexing(true); - auto uPtrBb = funcgrpc::SerializeToByteBuffer(&streamingMsg); - auto byteBuffer = uPtrBb.get(); - - FUNC_LOG_DEBUG("Pushing response to outbound channel.contentCase: {}", streamingMsg.content_case()); - funcgrpc::MessageChannel::GetInstance().GetOutboundChannel().push(*byteBuffer); - } - else if (contentCase == StreamingMessage::ContentCase::kFunctionEnvironmentReloadRequest) - { - try - { - string dir(receivedMessage.function_environment_reload_request().function_app_directory()); - - { - funcgrpc::FuncPerfMarker mark1("Setting environment variables"); - - google::protobuf::Map envVars = - receivedMessage.function_environment_reload_request().environment_variables(); - for (auto &envVar : envVars) - { - string envString = envVar.first; // key - string value = envVar.second; // value - envString.append("=").append(value); - - _putenv(envString.c_str()); - } - } - - string exePath = funcgrpc::WorkerConfigHandle().GetApplicationExePath(dir); - { - funcgrpc::FuncPerfMarker mark2("application_->ExecuteApplication"); - application_->ExecuteApplication(exePath); - } - - FUNC_LOG_INFO("Waiting for worker initialization."); - std::unique_lock lk(application_->mtx_workerLoaded); - application_->cv_workerLoaded.wait(lk, [this] { return application_->hasWorkerLoaded; }); - FUNC_LOG_INFO("Worker payload loaded. Forwarding env reload request to worker."); - - funcgrpc::MessageChannel::GetInstance().GetInboundChannel().push(*receivedMessageBb); - specializationRequestReceived = true; - } - catch (const std::exception &ex) - { - FUNC_LOG_ERROR("Caught unknown exception inside handler.{}", ex.what()); - } - catch (...) - { - FUNC_LOG_ERROR("Caught unknown exception in handler."); - } - } - } -} \ No newline at end of file diff --git a/host/src/funcgrpc/handlers/funcgrpc_native_handler.h b/host/src/funcgrpc/handlers/funcgrpc_native_handler.h deleted file mode 100644 index d8e120ad6..000000000 --- a/host/src/funcgrpc/handlers/funcgrpc_native_handler.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#ifndef FUNC_NATIVEHANDLER -#define FUNC_NATIVEHANDLER - -#include "../funcgrpc.h" -#include "../funcgrpc_handlers.h" -#include "../nativehostapplication.h" -#include "grpcpp/support/byte_buffer.h" -#include -#include -#include -#include -#include -#include - -using namespace AzureFunctionsRpc; - -using grpc::ByteBuffer; -namespace AzureFunctionsRpc -{ - -class NativeHostMessageHandler : public MessageHandler -{ - - public: - explicit NativeHostMessageHandler(NativeHostApplication *application); - - void HandleMessage(ByteBuffer *receivedMessage) override; - - private: - NativeHostApplication *application_; - bool specializationRequestReceived = false; -}; -} // namespace AzureFunctionsRpc - -#endif \ No newline at end of file diff --git a/host/src/funcgrpc/messaging_channel.cpp b/host/src/funcgrpc/messaging_channel.cpp deleted file mode 100644 index 77d91cde5..000000000 --- a/host/src/funcgrpc/messaging_channel.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#include "messaging_channel.h" -#include - -namespace funcgrpc -{ -std::unique_ptr outboundChannelPtr = std::make_unique(); -std::unique_ptr inboundChannelPtr = std::make_unique(); - -MessageChannel &MessageChannel::GetInstance() -{ - static MessageChannel single; - return single; -} - -channel_t &MessageChannel::GetOutboundChannel() -{ - return *outboundChannelPtr; -} - -channel_t &MessageChannel::GetInboundChannel() -{ - return *inboundChannelPtr; -} -} // namespace funcgrpc \ No newline at end of file diff --git a/host/src/funcgrpc/messaging_channel.h b/host/src/funcgrpc/messaging_channel.h deleted file mode 100644 index 5c31849f7..000000000 --- a/host/src/funcgrpc/messaging_channel.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#pragma once -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -using grpc::ByteBuffer; - -namespace funcgrpc -{ -typedef boost::fibers::unbuffered_channel channel_t; -typedef boost::fibers::channel_op_status channel_pop_status_t; - -class MessageChannel -{ - private: - MessageChannel() = default; - - public: - static MessageChannel &GetInstance(); - - /// - /// Gets the outbound channel. Any messages which needs to go out (to the host) - /// should be pushed to this channel. - /// - /// - channel_t &GetOutboundChannel(); - - /// - /// Gets the inbound channel. - /// Call pop() on this channel to get the messages coming to the worker from host. - /// Invocation request is an example message coming through this channel. - /// Example use: - /// StreamingMessage message; - /// while (channel_pop_status_t::success == outboundChannel.pop(message)) { - /// // Do something with the message - /// } - /// - /// - channel_t &GetInboundChannel(); -}; -} // namespace funcgrpc \ No newline at end of file diff --git a/host/src/funcgrpc/nativehostapplication.cpp b/host/src/funcgrpc/nativehostapplication.cpp deleted file mode 100644 index 2d6ae0a5c..000000000 --- a/host/src/funcgrpc/nativehostapplication.cpp +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#include "nativehostapplication.h" -#include "func_log.h" -#include - -using namespace std; - -NativeHostApplication *NativeHostApplication::s_Application = nullptr; - -NativeHostApplication::NativeHostApplication() -{ - initMutex_ = CreateMutex(nullptr, FALSE, nullptr); - load_hostfxr(); -} - -NativeHostApplication::~NativeHostApplication() -{ -} - -void NativeHostApplication::ExecuteApplication(string dllPath) -{ - s_Application = this; - - hostfxr_handle cxt = nullptr; - - wstring wdllPath(dllPath.begin(), dllPath.end()); - - const char_t *dotnet_app = wdllPath.c_str(); - int rc = init_fptr(1, &dotnet_app, nullptr, &cxt); - - if (rc != 0 || cxt == nullptr) - { - std::cerr << "Init failed: " << std::hex << std::showbase << rc << '\n'; - close_fptr(cxt); - } - - set_runtime_prop(cxt, L"AZURE_FUNCTIONS_NATIVE_HOST", L"1"); - - clrThread_ = thread( - [](hostfxr_run_app_fn r, hostfxr_handle h, HANDLE m) { - WaitForSingleObject(m, INFINITE); - - int rc = r(h); - - if (rc != 0 || h == nullptr) - { - std::cerr << "Init failed2: " << std::hex << std::showbase << rc << '\n'; - // close_fptr(cxt); - } - }, - run_app_fptr, cxt, initMutex_); -} - -void NativeHostApplication::HandleIncomingMessage(unsigned char *buffer, int size) -{ - WaitForSingleObject(initMutex_, INFINITE); - - callback(&buffer, size, handle); -} - -void NativeHostApplication::SendOutgoingMessage(_In_ ByteBuffer *msg) -{ - FUNC_LOG_DEBUG("NativeHostApplication::SendOutgoingMessage > Pushing message to outbound channel."); - auto &outboundChannel = funcgrpc::MessageChannel::GetInstance().GetOutboundChannel(); - outboundChannel.push(*msg); -} - -void NativeHostApplication::SetCallbackHandles(_In_ PFN_REQUEST_HANDLER request_callback, _In_ void *grpcHandle) -{ - callback = request_callback; - handle = grpcHandle; - - ReleaseMutex(initMutex_); - - { - std::lock_guard lk(mtx_workerLoaded); - hasWorkerLoaded = true; - } - - cv_workerLoaded.notify_one(); -} - -bool NativeHostApplication::load_hostfxr() -{ - // Pre-allocate a large buffer for the path to hostfxr - char_t buffer[MAX_PATH]; - size_t buffer_size = sizeof(buffer) / sizeof(char_t); - int rc = get_hostfxr_path(buffer, &buffer_size, nullptr); - if (rc != 0) - return false; - - // Load hostfxr and get desired exports - void *lib = load_library(buffer); - - init_fptr = - (hostfxr_initialize_for_dotnet_command_line_fn)get_export(lib, "hostfxr_initialize_for_dotnet_command_line"); - get_delegate_fptr = (hostfxr_get_runtime_delegate_fn)get_export(lib, "hostfxr_get_runtime_delegate"); - set_runtime_prop = (hostfxr_set_runtime_property_value_fn)get_export(lib, "hostfxr_set_runtime_property_value"); - run_app_fptr = (hostfxr_run_app_fn)get_export(lib, "hostfxr_run_app"); - close_fptr = (hostfxr_close_fn)get_export(lib, "hostfxr_close"); - - return (init_fptr && get_delegate_fptr && close_fptr); -} - -void *NativeHostApplication::load_library(const char_t *path) -{ - HMODULE h = ::LoadLibraryW(path); - assert(h != nullptr); - return (void *)h; -} - -void *NativeHostApplication::get_export(void *h, const char *name) -{ - void *f = ::GetProcAddress((HMODULE)h, name); - assert(f != nullptr); - return f; -} \ No newline at end of file diff --git a/host/src/funcgrpc/nativehostapplication.h b/host/src/funcgrpc/nativehostapplication.h deleted file mode 100644 index 68b052036..000000000 --- a/host/src/funcgrpc/nativehostapplication.h +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -#pragma once -#include "messaging_channel.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -using grpc::ByteBuffer; -using namespace std; - -// delegate for requests -typedef int(__stdcall *PFN_REQUEST_HANDLER)(unsigned char **msg, int size, void *grpcHandle); - -class NativeHostApplication -{ - public: - NativeHostApplication(); - - ~NativeHostApplication(); - - void ExecuteApplication(string dllPath); - - void SetCallbackHandles(_In_ PFN_REQUEST_HANDLER request_callback, _In_ void *grpcHandle); - - void HandleIncomingMessage(_In_ unsigned char *buffer, _In_ int size); - - void SendOutgoingMessage(_In_ ByteBuffer *message); - - static NativeHostApplication *GetInstance() - { - return s_Application; - } - - // Indicates whether the worker payload(managed code) loaded. - bool hasWorkerLoaded = false; - - std::condition_variable cv_workerLoaded; - - std::mutex mtx_workerLoaded; - private: - static NativeHostApplication *s_Application; - - // Globals to hold hostfxr exports - hostfxr_initialize_for_dotnet_command_line_fn init_fptr; - hostfxr_get_runtime_delegate_fn get_delegate_fptr; - hostfxr_set_runtime_property_value_fn set_runtime_prop; - hostfxr_run_app_fn run_app_fptr; - hostfxr_close_fn close_fptr; - - bool load_hostfxr(); - void *load_library(const char_t *); - void *get_export(void *h, const char *name); - - PFN_REQUEST_HANDLER callback; - void *handle; - - thread clrThread_; - HANDLE initMutex_; -}; \ No newline at end of file diff --git a/host/src/vcpkg.json b/host/src/vcpkg.json deleted file mode 100644 index bbd0d3c30..000000000 --- a/host/src/vcpkg.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg/master/scripts/vcpkg.schema.json", - "name": "func-net-host", - "version": "0.0.1", - "builtin-baseline": "6f7ffeb18f99796233b958aaaf14ec7bd4fb64b2", - "dependencies": [ - "grpc", - "boost-program-options", - "boost-fiber", - "spdlog", - "nethost", - "rapidjson" - ] -} \ No newline at end of file diff --git a/host/tools/build/Microsoft.Azure.Functions.DotnetIsolatedNativeHost.nuspec b/host/tools/build/Microsoft.Azure.Functions.DotnetIsolatedNativeHost.nuspec index 76e84eac7..2fe6ae20c 100644 --- a/host/tools/build/Microsoft.Azure.Functions.DotnetIsolatedNativeHost.nuspec +++ b/host/tools/build/Microsoft.Azure.Functions.DotnetIsolatedNativeHost.nuspec @@ -4,7 +4,7 @@ Microsoft.Azure.Functions.DotNetIsolatedNativeHost Microsoft Azure Functions dotnet-isolated native host dotnet-isolated azure-functions azure - 1.0.0-preview8 + 1.0.0-preview805 Microsoft Microsoft https://github.com/Azure/azure-functions-dotnet-worker @@ -17,7 +17,8 @@ - + + \ No newline at end of file diff --git a/host/tools/build/worker.config.json b/host/tools/build/worker.config.json index 3b248f8cd..7dfab24f9 100644 --- a/host/tools/build/worker.config.json +++ b/host/tools/build/worker.config.json @@ -5,5 +5,21 @@ "defaultExecutablePath": "%FUNCTIONS_WORKER_DIRECTORY%/bin/FunctionsNetHost.exe", "defaultWorkerPath": "bin/FunctionsNetHost.exe", "workerIndexing": "true" - } + }, + "profiles": [ + { + "profileName": "DotnetIsolatedLinux", + "conditions": [ + { + "conditionType": "hostProperty", + "conditionName": "platform", + "conditionExpression": "LINUX" + } + ], + "description": { + "defaultExecutablePath": "%FUNCTIONS_WORKER_DIRECTORY%/bin/FunctionsNetHost", + "defaultWorkerPath": "bin/FunctionsNetHost" + } + } + ] } \ No newline at end of file diff --git a/release_notes.md b/release_notes.md index c96b86559..540e00977 100644 --- a/release_notes.md +++ b/release_notes.md @@ -14,4 +14,4 @@ ### Microsoft.Azure.Functions.Worker.Grpc -- +- Add placeholder support for linux platform. (#1704) \ No newline at end of file diff --git a/src/DotNetWorker.Grpc/DotNetWorker.Grpc.csproj b/src/DotNetWorker.Grpc/DotNetWorker.Grpc.csproj index 59cf3970d..df6d0db63 100644 --- a/src/DotNetWorker.Grpc/DotNetWorker.Grpc.csproj +++ b/src/DotNetWorker.Grpc/DotNetWorker.Grpc.csproj @@ -2,7 +2,7 @@ Library - net5.0;netstandard2.0 + net5.0;net6.0;net7.0;netstandard2.0 Microsoft.Azure.Functions.Worker.Grpc This library provides gRPC support for Azure Functions .NET Worker communication with the Azure Functions Host. Microsoft.Azure.Functions.Worker.Grpc @@ -45,10 +45,7 @@ - + diff --git a/src/DotNetWorker.Grpc/NativeHostIntegration/NativeMethods.cs b/src/DotNetWorker.Grpc/NativeHostIntegration/NativeMethods.cs index 7d0e03b5c..390d45efe 100644 --- a/src/DotNetWorker.Grpc/NativeHostIntegration/NativeMethods.cs +++ b/src/DotNetWorker.Grpc/NativeHostIntegration/NativeMethods.cs @@ -8,14 +8,24 @@ namespace Microsoft.Azure.Functions.Worker.Grpc.NativeHostIntegration { - internal static unsafe partial class NativeMethods + internal static unsafe class NativeMethods { private const string NativeWorkerDll = "FunctionsNetHost.exe"; + static NativeMethods() + { + NativeLibrary.SetDllImportResolver(typeof(NativeMethods).Assembly, ImportResolver); + } + public static NativeHost GetNativeHostData() { - _ = get_application_properties(out var hostData); - return hostData; + var result = get_application_properties(out var hostData); + if (result == 1) + { + return hostData; + } + + throw new InvalidOperationException($"Invalid result returned from get_application_properties: {result}"); } public static void RegisterCallbacks(NativeSafeHandle nativeApplication, @@ -31,15 +41,39 @@ public static void SendStreamingMessage(NativeSafeHandle nativeApplication, Stre _ = send_streaming_message(nativeApplication, bytes, bytes.Length); } - [DllImport(NativeWorkerDll)] + [DllImport(NativeWorkerDll, CharSet = CharSet.Auto)] private static extern int get_application_properties(out NativeHost hostData); - [DllImport(NativeWorkerDll)] + [DllImport(NativeWorkerDll, CharSet = CharSet.Auto)] private static extern int send_streaming_message(NativeSafeHandle pInProcessApplication, byte[] streamingMessage, int streamingMessageSize); - [DllImport(NativeWorkerDll)] + [DllImport(NativeWorkerDll, CharSet = CharSet.Auto)] private static extern unsafe int register_callbacks(NativeSafeHandle pInProcessApplication, delegate* unmanaged requestCallback, IntPtr grpcHandler); + + /// + /// Custom import resolve callback. + /// When trying to resolve "FunctionsNetHost", we return the handle using GetMainProgramHandle API in this callback. + /// + private static IntPtr ImportResolver(string libraryName, System.Reflection.Assembly assembly, DllImportSearchPath? searchPath) + { + if (libraryName == NativeWorkerDll) + { +#if NET6_0 + if (OperatingSystem.IsLinux()) + { + return NativeLibraryLinux.GetMainProgramHandle(); + } +#elif NET7_0_OR_GREATER + return NativeLibrary.GetMainProgramHandle(); +#else + throw new PlatformNotSupportedException("Interop communication with FunctionsNetHost is not supported in the current platform. Consider upgrading your project to .NET 7.0 or later."); +#endif + } + + // Return 0 so that built-in resolving code will be executed. + return IntPtr.Zero; + } } } diff --git a/src/DotNetWorker.Grpc/NativeHostIntegration/Shim/NativeLibrary.Linux.cs b/src/DotNetWorker.Grpc/NativeHostIntegration/Shim/NativeLibrary.Linux.cs new file mode 100644 index 000000000..4166785fa --- /dev/null +++ b/src/DotNetWorker.Grpc/NativeHostIntegration/Shim/NativeLibrary.Linux.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.Azure.Functions.Worker.Grpc.NativeHostIntegration +{ + /// + /// NativeLibrary.GetMainProgramHandle is only available from NET7. + /// This shim calls the native API on Linux to get the main program handle + /// + internal class NativeLibraryLinux + { + // Value 1 loads the library lazily, resolving symbols only as they are used + private const int RTLD_LAZY = 1; + + [DllImport("libdl.so", CharSet = CharSet.Auto)] + private static extern IntPtr dlerror(); + + [DllImport("libdl.so", CharSet = CharSet.Auto)] + private static extern IntPtr dlclose(nint handle); + + [DllImport("libdl.so", CharSet = CharSet.Auto)] + private static extern IntPtr dlopen(string filename, int flags); + + internal static IntPtr GetMainProgramHandle() + { +#pragma warning disable CS8625 // Passing null will return main program handle. + var handle = dlopen(filename: null, RTLD_LAZY); +#pragma warning restore CS8625 + + if (handle == IntPtr.Zero) + { + var error = Marshal.PtrToStringAnsi(dlerror()); + throw new InvalidOperationException($"Failed to get main program handle.{error}"); + } + + var result = dlclose(handle); + if (result != IntPtr.Zero) + { + var error = Marshal.PtrToStringAnsi(dlerror()); + throw new InvalidOperationException($"Failed to close main program handle: {error}"); + } + + return handle; + } + } +} diff --git a/src/DotNetWorker/DotNetWorker.csproj b/src/DotNetWorker/DotNetWorker.csproj index 36bc0e0ee..a4d0fc1dc 100644 --- a/src/DotNetWorker/DotNetWorker.csproj +++ b/src/DotNetWorker/DotNetWorker.csproj @@ -2,7 +2,7 @@ Library - net5.0;netstandard2.0 + net5.0;net6.0;net7.0;netstandard2.0 Microsoft.Azure.Functions.Worker This library enables you to create an Azure Functions .NET Worker, adding support for the isolated, out-of-process execution model. Microsoft.Azure.Functions.Worker From d40a074e2432b5b1d3ecbfd3ec67dbd46eba220a Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 26 Jul 2023 15:09:59 -0700 Subject: [PATCH 46/47] Refactor analyzer BindingTypeCodeRefactoringProviderTests (#1783) --- .../Extensions/TypeSymbolExtensions.cs | 5 + ...BindingTypeCodeRefactoringProviderTests.cs | 256 ++++++++++++------ .../Sdk.Analyzers.Tests.csproj | 4 +- 3 files changed, 176 insertions(+), 89 deletions(-) diff --git a/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs b/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs index 246538acb..5a6581bc1 100644 --- a/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs +++ b/sdk/Sdk.Analyzers/Extensions/TypeSymbolExtensions.cs @@ -58,6 +58,11 @@ internal static string GetMinimalDisplayName(this ITypeSymbol type, SemanticMode name = Regex.Match(name, @"IEnumerable<[^>]+>").Value; } + if (name.StartsWith("System.")) + { + name = Regex.Match(name, @"([^.]+$)").Value; + } + return name; } diff --git a/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs b/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs index 2394001b7..ec0a5ab76 100644 --- a/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs +++ b/test/Sdk.Analyzers.Tests/BindingTypeCodeRefactoringProviderTests.cs @@ -9,8 +9,6 @@ namespace Sdk.Analyzers.Tests { - // Disabling tests as they depend on Tables and ServiceBus extension releases with new deferred binding model changes - // Issue #1746 created to re-enable tests once new releases are available public class BindingTypeCodeRefactoringProviderTests : CodeRefactoringTestFixture { protected override string LanguageName => LanguageNames.CSharp; @@ -24,94 +22,178 @@ protected override CodeRefactoringProvider CreateProvider() { ReferenceSource.NetStandard2_0, ReferenceSource.FromAssembly(Assembly.Load("System.Runtime, Version=7.0.0.0").Location), - ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Core, Version=1.13.0.0").Location), - ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.Abstractions, Version=1.2.0.0")), + ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Core, Version=1.14.0.0").Location), + ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.Abstractions, Version=1.3.0.0")), - ReferenceSource.FromAssembly(Assembly.Load("Azure.Data.Tables, Version=12.8.0.0").Location), - ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.Tables, Version=1.2.0.0").Location), + ReferenceSource.FromAssembly(Assembly.Load("Azure.Storage.Queues, Version=12.13.1.0").Location), + ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues, Version=5.2.0.0").Location), - ReferenceSource.FromAssembly(Assembly.Load("Azure.Messaging.ServiceBus, Version=7.14.0.0").Location), - ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.ServiceBus, Version=5.10.0.0").Location), + ReferenceSource.FromAssembly(Assembly.Load("Azure.Messaging.EventHubs, Version=5.9.2.0").Location), + ReferenceSource.FromAssembly(Assembly.Load("Microsoft.Azure.Functions.Worker.Extensions.EventHubs, Version=5.5.0.0").Location) }; - // [Theory] - // [InlineData("TableClient", 0)] - // [InlineData("TableEntity", 1)] - // public void TableInput_SuggestsCodeRefactor(string supportedType, int index) - // { - // string testCode = @" - // using System; - // using Azure.Data.Tables; - // using Microsoft.Azure.Functions.Worker; - - // namespace FunctionApp - // { - // public static class SomeFunction - // { - // [Function(nameof(SomeFunction))] - // public static void Run([TableInput(""input"")] [|string message|]) - // { - // } - // } - // }"; - - // string expectedCode = $@" - // using System; - // using Azure.Data.Tables; - // using Microsoft.Azure.Functions.Worker; - - // namespace FunctionApp - // {{ - // public static class SomeFunction - // {{ - // [Function(nameof(SomeFunction))] - // public static void Run([TableInput(""input"")] {supportedType} message) - // {{ - // }} - // }} - // }}"; - - // TestCodeRefactoring(testCode, expectedCode, index); - // } - - // [Theory] - // [InlineData("ServiceBusReceivedMessage", 0)] - // [InlineData("ServiceBusReceivedMessage[]", 1)] - // public void ServiceBusTrigger_SuggestsCodeRefactor(string supportedType, int index) - // { - // string testCode = @" - // using System; - // using Azure.Messaging.ServiceBus; - // using Microsoft.Azure.Functions.Worker; - - // namespace FunctionApp - // { - // public static class SomeFunction - // { - // [Function(nameof(SomeFunction))] - // public static void Run([ServiceBusTrigger(""input"")] [|string message|]) - // { - // } - // } - // }"; - - // string expectedCode = $@" - // using System; - // using Azure.Messaging.ServiceBus; - // using Microsoft.Azure.Functions.Worker; - - // namespace FunctionApp - // {{ - // public static class SomeFunction - // {{ - // [Function(nameof(SomeFunction))] - // public static void Run([ServiceBusTrigger(""input"")] {supportedType} message) - // {{ - // }} - // }} - // }}"; - - // TestCodeRefactoring(testCode, expectedCode, index); - // } + [Theory] + [InlineData("string", 0)] + [InlineData("bool", 1)] + [InlineData("IEnumerable", 2)] + [InlineData("bool[]", 3)] + public void TestBinding_SuggestsCodeRefactor(string supportedType, int index) + { + string testCode = @" + using System; + using System.Collections.Generic; + using Microsoft.Azure.Functions.Worker; + using Microsoft.Azure.Functions.Worker.Core; + using Microsoft.Azure.Functions.Worker.Converters; + using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + + namespace Microsoft.Azure.Functions.Worker + { + [ConverterFallbackBehavior(ConverterFallbackBehavior.Disallow)] + [InputConverter(typeof(TestConverter))] + public class TestTriggerAttribute : TriggerBindingAttribute + { + } + + [SupportsDeferredBinding] + [SupportedTargetType(typeof(string))] + [SupportedTargetType(typeof(bool))] + [SupportedTargetType(typeof(IEnumerable))] + [SupportedTargetType(typeof(bool[]))] + public class TestConverter + { + } + } + + namespace FunctionApp + { + public class SomeFunction + { + [Function(nameof(SomeFunction))] + public void Run([TestTrigger()] [|int message|]) + { + } + } + }"; + + string expectedCode = $@" + using System; + using System.Collections.Generic; + using Microsoft.Azure.Functions.Worker; + using Microsoft.Azure.Functions.Worker.Core; + using Microsoft.Azure.Functions.Worker.Converters; + using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + + namespace Microsoft.Azure.Functions.Worker + {{ + [ConverterFallbackBehavior(ConverterFallbackBehavior.Disallow)] + [InputConverter(typeof(TestConverter))] + public class TestTriggerAttribute : TriggerBindingAttribute + {{ + }} + + [SupportsDeferredBinding] + [SupportedTargetType(typeof(string))] + [SupportedTargetType(typeof(bool))] + [SupportedTargetType(typeof(IEnumerable))] + [SupportedTargetType(typeof(bool[]))] + public class TestConverter + {{ + }} + }} + + namespace FunctionApp + {{ + public class SomeFunction + {{ + [Function(nameof(SomeFunction))] + public void Run([TestTrigger()] {supportedType} message) + {{ + }} + }} + }}"; + + TestCodeRefactoring(testCode, expectedCode, index); + } + + [Theory] + [InlineData("QueueMessage", 0)] + [InlineData("BinaryData", 1)] + public void QueueTrigger_SuggestsCodeRefactor(string supportedType, int index) + { + string testCode = @" + using System; + using Azure.Storage.Queues.Models; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([QueueTrigger(""input"")] [|string message|]) + { + } + } + }"; + + string expectedCode = $@" + using System; + using Azure.Storage.Queues.Models; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + {{ + public static class SomeFunction + {{ + [Function(nameof(SomeFunction))] + public static void Run([QueueTrigger(""input"")] {supportedType} message) + {{ + }} + }} + }}"; + + TestCodeRefactoring(testCode, expectedCode, index); + } + + [Theory] + [InlineData("EventData", 0)] + [InlineData("EventData[]", 1)] + public void EventHubTrigger_SuggestsCodeRefactor(string supportedType, int index) + { + string testCode = @" + using System; + using Azure.Messaging.EventHubs; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + { + public static class SomeFunction + { + [Function(nameof(SomeFunction))] + public static void Run([EventHubTrigger(""input"")] [|string message|]) + { + } + } + }"; + + string expectedCode = $@" + using System; + using Azure.Messaging.EventHubs; + using Microsoft.Azure.Functions.Worker; + + namespace FunctionApp + {{ + public static class SomeFunction + {{ + [Function(nameof(SomeFunction))] + public static void Run([EventHubTrigger(""input"")] {supportedType} message) + {{ + }} + }} + }}"; + + TestCodeRefactoring(testCode, expectedCode, index); + } } } diff --git a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj b/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj index 5c84f5ced..13e5a9c7a 100644 --- a/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj +++ b/test/Sdk.Analyzers.Tests/Sdk.Analyzers.Tests.csproj @@ -24,8 +24,8 @@ - - + + From c7b97955caf827493d223020887c2ce4c4ad539e Mon Sep 17 00:00:00 2001 From: Lilian Kasem Date: Wed, 26 Jul 2023 15:57:17 -0700 Subject: [PATCH 47/47] Bump Sdk to 1.13.0 (#1788) --- sdk/Sdk/Sdk.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/Sdk/Sdk.csproj b/sdk/Sdk/Sdk.csproj index 686ad473e..8b8ef3ac9 100644 --- a/sdk/Sdk/Sdk.csproj +++ b/sdk/Sdk/Sdk.csproj @@ -1,7 +1,7 @@  - 12 + 13 0 netstandard2.0;net472 Microsoft.Azure.Functions.Worker.Sdk