From e9d6489787a5ea5400a31dfa34aa6ad6b590de9b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:56:05 +0000 Subject: [PATCH 01/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20230913.1 (#212) [release/8.0] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index da38c99cd5..a9a4b159b2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 4665b3d04e1da3796b965c3c3e3b97f55c449a6e - + https://github.com/dotnet/source-build-reference-packages - 18302345498a62222dd3c236b3d1d140dbaed43d + 3dd2c0ef203db8fe0e849557960b4cd009afbaac From c42a7ce3b6fa02957e7b4ef995c5c2a9a23d294c Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Tue, 19 Sep 2023 14:58:23 -0700 Subject: [PATCH 02/33] Add xunit.abstractions back (#209) Took a slightly tweaked approach. For projects that have dependencies, the nuget config files are not deleted. Instead they are cleared, and the package outputs from the dependencies are added. This approach appears to be the easiest. The other alternative was to copy the package outputs of dependencies into the shared nuget cache used for source-build-intermediates. The problem is that this location is not actually known to the outer source build where the build orchestration takes place. That's only known to the inner source build infra. --- .gitmodules | 3 + keys/xunit.abstractions.snk | Bin 0 -> 596 bytes .../abstractions-xunit/0001-Patching.patch | 61 ++++++++++++++++++ repo-projects/Directory.Build.targets | 61 ++++++++++++++++-- repo-projects/abstractions-xunit.proj | 41 ++++++++++++ repo-projects/xunit.proj | 4 ++ src/abstractions-xunit | 1 + 7 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 keys/xunit.abstractions.snk create mode 100644 patches/abstractions-xunit/0001-Patching.patch create mode 100644 repo-projects/abstractions-xunit.proj create mode 160000 src/abstractions-xunit diff --git a/.gitmodules b/.gitmodules index f99d465f7a..19152635c4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ [submodule "xunit"] path = src/xunit url = https://github.com/xunit/xunit +[submodule "abstractions-xunit"] + path = src/abstractions-xunit + url = https://github.com/xunit/abstractions.xunit diff --git a/keys/xunit.abstractions.snk b/keys/xunit.abstractions.snk new file mode 100644 index 0000000000000000000000000000000000000000..93641b9761c08f6b592aba340776edc14a5ec27d GIT binary patch literal 596 zcmV-a0;~N80ssI2Bme+XQ$aES1ONa50096cE(Dt0>WA|Wnbz%&ycnCnTa1xtc3tp~ zhTtJYaomZ7PajL1N+yX-0+eNhhjF}hkL@5g5`gWZBdPXE;9p3yAms4k3_^n4a2}Xf z5xg_Cx#p`sud8ggjv@}?xZza=bKpOU)@8s^#y`AMS=5!(vBnT2?{@8+9=k^$?ODm$ zWzI_^s25nW&CvR}S&P!2&ZB4N_L#Bxt@EFD=y5Fj=s4UxLB%ePo^~{;)v(Tyx_B6H;!$eo`L}NQ8Hjnl$HRmpx7?cObLFYs zB*ORoG(%TSO@WQSzsG^hUUVV}m&v62&_vu^YSumrlYBl~dZs@`S2y_v zlAg!IY~H)v;BzX&THDP^!87KeV^ZJ&J87PMX}O^B)9Oop7vK5U_XVQg`9v;vM9Px^ zoj3JO}ZN;?jXV=KovW8!n$BBD25zO*C8fL3H5eJ^OCXCEAGC@JE zZfPW78wpVX&NicmbZ!=5_AN&=L3C7_9Gqvg`l2~J$2vW>MD8WY4XIy3@l)ze%64QI i@n`^|4l~$$Df`THrXhhM*WhbjYjkP&G#gZfW1qZXj4Ag3 literal 0 HcmV?d00001 diff --git a/patches/abstractions-xunit/0001-Patching.patch b/patches/abstractions-xunit/0001-Patching.patch new file mode 100644 index 0000000000..3d0be07f74 --- /dev/null +++ b/patches/abstractions-xunit/0001-Patching.patch @@ -0,0 +1,61 @@ +From b68777acde280b91039b98ff36c4d028064a8f52 Mon Sep 17 00:00:00 2001 +From: Matt Mitchell +Date: Wed, 19 Jul 2023 11:47:26 -0700 +Subject: [PATCH] Patching + +--- + global.json | 6 ------ + src/xunit.abstractions/Properties/AssemblyInfo.cs | 2 +- + src/xunit.abstractions/xunit.abstractions.csproj | 11 +++-------- + 3 files changed, 4 insertions(+), 15 deletions(-) + delete mode 100644 global.json + +diff --git a/global.json b/global.json +deleted file mode 100644 +index 9f78419..0000000 +--- a/global.json ++++ /dev/null +@@ -1,6 +0,0 @@ +-{ +- "sdk": { +- "version": "6.0.0", +- "rollForward": "latestMinor" +- } +-} +diff --git a/src/xunit.abstractions/Properties/AssemblyInfo.cs b/src/xunit.abstractions/Properties/AssemblyInfo.cs +index 041f22b..c5b5124 100644 +--- a/src/xunit.abstractions/Properties/AssemblyInfo.cs ++++ b/src/xunit.abstractions/Properties/AssemblyInfo.cs +@@ -17,6 +17,6 @@ + [assembly: AssemblyProduct("xUnit.net Testing Framework")] + [assembly: AssemblyCopyright("Copyright (C) .NET Foundation")] + [assembly: CLSCompliant(true)] +-[assembly: AssemblyVersion("2.0.1.0")] ++[assembly: AssemblyVersion("2.0.3.0")] + + [assembly: SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "xunit")] +diff --git a/src/xunit.abstractions/xunit.abstractions.csproj b/src/xunit.abstractions/xunit.abstractions.csproj +index fa4cf23..5d61131 100644 +--- a/src/xunit.abstractions/xunit.abstractions.csproj ++++ b/src/xunit.abstractions/xunit.abstractions.csproj +@@ -9,14 +9,9 @@ + false + false + false +- net35;netstandard1.0 ++ true ++ $(PackageVersion) ++ netstandard2.0 + + +- +- +- runtime; build; native; contentfiles; analyzers; buildtransitive +- all +- +- +- + +-- +2.41.0.windows.2 + diff --git a/repo-projects/Directory.Build.targets b/repo-projects/Directory.Build.targets index 916e2c093e..aff7be9f8b 100644 --- a/repo-projects/Directory.Build.targets +++ b/repo-projects/Directory.Build.targets @@ -7,6 +7,8 @@ $(ProjectDirectory)src\NuGet.Config + + @@ -47,16 +49,64 @@ found instead. This lets the subrepo use the incoming intermediate nupkg contents and avoids the need to do subrepo-specific fixups like removing internet sources. --> - + + + + + + + + + $(ProjectDirectory)NuGet.config + + + + <_DependentProjectSources Include="@(RepositoryReference -> '%(Identity).proj')"> + %(Identity) + + + + + + + + + + + + + + +]]> + + + + + + + + + @@ -185,6 +235,7 @@ + diff --git a/repo-projects/abstractions-xunit.proj b/repo-projects/abstractions-xunit.proj new file mode 100644 index 0000000000..9c3e230b46 --- /dev/null +++ b/repo-projects/abstractions-xunit.proj @@ -0,0 +1,41 @@ + + + + + $(ProjectDirectory)/src/xunit.abstractions/bin/$(Configuration)/ + $(KeysDir)xunit.abstractions.snk + + + + + + + $(ProjectDirectory)/xunit.abstractions.sln + $(BuildCommandArgs) /p:Configuration=$(Configuration) + $(BuildCommandArgs) /p:PackageVersion=2.0.3 + $(BuildCommandArgs) /p:AssemblyOriginatorKeyFile=$(XunitAbstractionsKeyFilePath) + $(BuildCommandArgs) /p:DelaySign=$(DelaySign) + $(BuildCommandArgs) /p:SignAssembly=true + $(BuildCommandArgs) /p:PublicSign=$(PublicSign) + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + + \ No newline at end of file diff --git a/repo-projects/xunit.proj b/repo-projects/xunit.proj index 47c856789d..bddb5ffc58 100644 --- a/repo-projects/xunit.proj +++ b/repo-projects/xunit.proj @@ -6,6 +6,10 @@ + + + + diff --git a/src/abstractions-xunit b/src/abstractions-xunit new file mode 160000 index 0000000000..b75d54d73b --- /dev/null +++ b/src/abstractions-xunit @@ -0,0 +1 @@ +Subproject commit b75d54d73b141709f805c2001b16f3dd4d71539d From 6dbf3aaa0fc9664df86462f5c70b99800934fccd Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Thu, 21 Sep 2023 12:32:10 -0500 Subject: [PATCH 03/33] MSBuildLocator 1.6.10 (#214) (#215) * DOTNET_ROOT handling changes and packages warning presence condition * MSBuild Locaton version update to 1.6.10 * Disable MSBuildLocator package validation This fails in sourcebuild because we overwrite the TF to current ones, but that's kinda the point of source-build so it's not a _problem_. --------- Co-authored-by: YuliiaKovalova --- repo-projects/MSBuildLocator.proj | 3 ++- src/MSBuildLocator | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/repo-projects/MSBuildLocator.proj b/repo-projects/MSBuildLocator.proj index 68eb28d3ec..f12a4a4064 100644 --- a/repo-projects/MSBuildLocator.proj +++ b/repo-projects/MSBuildLocator.proj @@ -15,7 +15,8 @@ $(BuildCommandArgs) $(RedirectRepoOutputToLog) $(BuildCommandArgs) /p:DelaySign=$(DelaySign) $(BuildCommandArgs) /p:PublicSign=$(PublicSign) - $(BuildCommandArgs) /p:Version=1.6.1 + $(BuildCommandArgs) /p:EnablePackageValidation=false + $(BuildCommandArgs) /p:Version=1.6.10 Date: Mon, 25 Sep 2023 12:36:20 +0000 Subject: [PATCH 04/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20230921.1 (#219) [release/8.0] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a9a4b159b2..5ac4ace5d1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 4665b3d04e1da3796b965c3c3e3b97f55c449a6e - + https://github.com/dotnet/source-build-reference-packages - 3dd2c0ef203db8fe0e849557960b4cd009afbaac + 7b55da982fc6e71c1776c4de89111aee0eecb45a From ed17956dbc31097b7ba6a66be086f4a70a97d84f Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Mon, 2 Oct 2023 07:38:02 -0500 Subject: [PATCH 05/33] Patch to remove WebGrease from TPN for app insights (#220) --- .../0002-Remove-WebGrease-from-TPN-2816.patch | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 patches/application-insights/0002-Remove-WebGrease-from-TPN-2816.patch diff --git a/patches/application-insights/0002-Remove-WebGrease-from-TPN-2816.patch b/patches/application-insights/0002-Remove-WebGrease-from-TPN-2816.patch new file mode 100644 index 0000000000..b516c82685 --- /dev/null +++ b/patches/application-insights/0002-Remove-WebGrease-from-TPN-2816.patch @@ -0,0 +1,156 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Matt Thalman +Date: Fri, 29 Sep 2023 16:06:46 -0500 +Subject: [PATCH] Remove WebGrease from TPN (#2816) + +Backport: https://github.com/microsoft/ApplicationInsights-dotnet/pull/2816 +--- + WEB/ThirdPartyNotices.txt | 128 +------------------------------------- + 1 file changed, 2 insertions(+), 126 deletions(-) + +diff --git a/WEB/ThirdPartyNotices.txt b/WEB/ThirdPartyNotices.txt +index b7a8964f..ea11d728 100644 +--- a/WEB/ThirdPartyNotices.txt ++++ b/WEB/ThirdPartyNotices.txt +@@ -28,9 +28,8 @@ The Visual Studio Application Insights SDK for .NET Web Applications source code + 22 Rx-Linq (https://github.com/Reactive-Extensions/Rx.NET) + 23 StyleCop.MSBuild (https://github.com/adamralph/stylecop-msbuild) + 24 System.Spatial (http://odata.github.io/) +-25 WebGrease (http://webgrease.codeplex.com/) +-26 WiX (http://wixtoolset.org/) +-27 xunit (https://github.com/xunit/xunit) ++25 WiX (http://wixtoolset.org/) ++26 xunit (https://github.com/xunit/xunit) + + + %% Antlr NOTICES, INFORMATION, AND LICENSE BEGIN HERE +@@ -498,129 +497,6 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI + ========================================= + END OF System.Spatial NOTICES, INFORMATION, AND LICENSE + +-%% WebGrease NOTICES, INFORMATION, AND LICENSE BEGIN HERE +-========================================= +-These license terms are an agreement between Microsoft Corporation (or based on where you live, one of its affiliates) and you. Please read them. They apply to the software named above, which includes the media on which you received it, if any. The terms also apply to any Microsoft +- +-· updates, +- +-· supplements, +- +-· Internet-based services, and +- +-· support services +- +-for this software, unless other terms accompany those items. If so, those terms apply. +- +-BY USING THE SOFTWARE, YOU ACCEPT THESE TERMS. IF YOU DO NOT ACCEPT THEM, DO NOT USE THE SOFTWARE. +- +-IF YOU COMPLY WITH THESE LICENSE TERMS, YOU HAVE THE PERPETUAL RIGHTS BELOW. +- +-1. INSTALLATION AND USE RIGHTS. +- +-a. Installation and Use. You may install and use any number of copies of the software to design, develop and test your programs. +- +-b. Third Party Programs. The software may include third party programs that Microsoft, not the third party, licenses to you under this agreement. Notices, if any, for the third party program are included for your information only. +- +-2. ADDITIONAL LICENSING REQUIREMENTS AND/OR USE RIGHTS. +- +-a. DISTRIBUTABLE CODE. The software is comprised of Distributable Code. “Distributable Code” is code that you are permitted to distribute in programs you develop if you comply with the terms below. +- +-i. Right to Use and Distribute. +- +-· You may copy and distribute the object code form of the software. +- +-· Third Party Distribution. You may permit distributors of your programs to copy and distribute the Distributable Code as part of those programs. +- +-ii. Distribution Requirements. For any Distributable Code you distribute, you must +- +-· add significant primary functionality to it in your programs; +- +-· require distributors and external end users to agree to terms that protect it at least as much as this agreement; +- +-· display your valid copyright notice on your programs; and +- +-· indemnify, defend, and hold harmless Microsoft from any claims, including attorneys’ fees, related to the distribution or use of your programs. +- +-iii. Distribution Restrictions. You may not +- +-· alter any copyright, trademark or patent notice in the Distributable Code; +- +-· use Microsoft’s trademarks in your programs’ names or in a way that suggests your programs come from or are endorsed by Microsoft; +- +-· include Distributable Code in malicious, deceptive or unlawful programs; or +- +-· modify or distribute the source code of any Distributable Code so that any part of it becomes subject to an Excluded License. An Excluded License is one that requires, as a condition of use, modification or distribution, that +- +-· the code be disclosed or distributed in source code form; or +- +-· others have the right to modify it. +- +-3. SCOPE OF LICENSE. The software is licensed, not sold. This agreement only gives you some rights to use the software. Microsoft reserves all other rights. Unless applicable law gives you more rights despite this limitation, you may use the software only as expressly permitted in this agreement. In doing so, you must comply with any technical limitations in the software that only allow you to use it in certain ways. You may not +- +-· work around any technical limitations in the software; +- +-· reverse engineer, decompile or disassemble the software, except and only to the extent that applicable law expressly permits, despite this limitation; +- +-· publish the software for others to copy; +- +-· rent, lease or lend the software; +- +-· transfer the software or this agreement to any third party; or +- +-· use the software for commercial software hosting services. +- +-4. BACKUP COPY. You may make one backup copy of the software. You may use it only to reinstall the software. +- +-5. DOCUMENTATION. Any person that has valid access to your computer or internal network may copy and use the documentation for your internal, reference purposes. +- +-6. EXPORT RESTRICTIONS. The software is subject to United States export laws and regulations. You must comply with all domestic and international export laws and regulations that apply to the software. These laws include restrictions on destinations, end users and end use. For additional information, see www.microsoft.com/exporting. +- +-7. SUPPORT SERVICES. Because this software is “as is,” we may not provide support services for it. +- +-8. ENTIRE AGREEMENT. This agreement, and the terms for supplements, updates, Internet-based services and support services that you use, are the entire agreement for the software and support services. +- +-9. APPLICABLE LAW. +- +-a. United States. If you acquired the software in the United States, Washington state law governs the interpretation of this agreement and applies to claims for breach of it, regardless of conflict of laws principles. The laws of the state where you live govern all other claims, including claims under state consumer protection laws, unfair competition laws, and in tort. +- +-b. Outside the United States. If you acquired the software in any other country, the laws of that country apply. +- +-10. LEGAL EFFECT. This agreement describes certain legal rights. You may have other rights under the laws of your country. You may also have rights with respect to the party from whom you acquired the software. This agreement does not change your rights under the laws of your country if the laws of your country do not permit it to do so. +- +-11. DISCLAIMER OF WARRANTY. THE SOFTWARE IS LICENSED “AS-IS.” YOU BEAR THE RISK OF USING IT. MICROSOFT GIVES NO EXPRESS WARRANTIES, GUARANTEES OR CONDITIONS. YOU MAY HAVE ADDITIONAL CONSUMER RIGHTS OR STATUTORY GUARANTEES UNDER YOUR LOCAL LAWS WHICH THIS AGREEMENT CANNOT CHANGE. TO THE EXTENT PERMITTED UNDER YOUR LOCAL LAWS, MICROSOFT EXCLUDES THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. +- +-FOR AUSTRALIA – YOU HAVE STATUTORY GUARANTEES UNDER THE AUSTRALIAN CONSUMER LAW AND NOTHING IN THESE TERMS IS INTENDED TO AFFECT THOSE RIGHTS. +- +-12. LIMITATION ON AND EXCLUSION OF REMEDIES AND DAMAGES. YOU CAN RECOVER FROM MICROSOFT AND ITS SUPPLIERS ONLY DIRECT DAMAGES UP TO U.S. $5.00. YOU CANNOT RECOVER ANY OTHER DAMAGES, INCLUDING CONSEQUENTIAL, LOST PROFITS, SPECIAL, INDIRECT OR INCIDENTAL DAMAGES. +- +-This limitation applies to +- +-· anything related to the software, services, content (including code) on third party Internet sites, or third party programs; and +- +-· claims for breach of contract, breach of warranty, guarantee or condition, strict liability, negligence, or other tort to the extent permitted by applicable law. +- +-It also applies even if Microsoft knew or should have known about the possibility of the damages. The above limitation or exclusion may not apply to you because your country may not allow the exclusion or limitation of incidental, consequential or other damages. +- +-Please note: As this software is distributed in Quebec, Canada, some of the clauses in this agreement are provided below in French. +- +-Remarque : Ce logiciel étant distribué au Québec, Canada, certaines des clauses dans ce contrat sont fournies ci-dessous en français. +- +-EXONÉRATION DE GARANTIE. Le logiciel visé par une licence est offert « tel quel ». Toute utilisation de ce logiciel est à votre seule risque et péril. Microsoft n’accorde aucune autre garantie expresse. Vous pouvez bénéficier de droits additionnels en vertu du droit local sur la protection des consommateurs, que ce contrat ne peut modifier. La ou elles sont permises par le droit locale, les garanties implicites de qualité marchande, d’adéquation à un usage particulier et d’absence de contrefaçon sont exclues. +- +-LIMITATION DES DOMMAGES-INTÉRÊTS ET EXCLUSION DE RESPONSABILITÉ POUR LES DOMMAGES. Vous pouvez obtenir de Microsoft et de ses fournisseurs une indemnisation en cas de dommages directs uniquement à hauteur de 5,00 $ US. Vous ne pouvez prétendre à aucune indemnisation pour les autres dommages, y compris les dommages spéciaux, indirects ou accessoires et pertes de bénéfices. +- +-Cette limitation concerne : +- +-· tout ce qui est relié au logiciel, aux services ou au contenu (y compris le code) figurant sur des sites Internet tiers ou dans des programmes tiers ; et +-· les réclamations au titre de violation de contrat ou de garantie, ou au titre de responsabilité stricte, de négligence ou d’une autre faute dans la limite autorisée par la loi en vigueur. +- +-Elle s’applique également, même si Microsoft connaissait ou devrait connaître l’éventualité d’un tel dommage. Si votre pays n’autorise pas l’exclusion ou la limitation de responsabilité pour les dommages indirects, accessoires ou de quelque nature que ce soit, il se peut que la limitation ou l’exclusion ci-dessus ne s’appliquera pas à votre égard. +- +-EFFET JURIDIQUE. Le présent contrat décrit certains droits juridiques. Vous pourriez avoir d’autres droits prévus par les lois de votre pays. Le présent contrat ne modifie pas les droits que vous confèrent les lois de votre pays si celles-ci ne le permettent pas. +-========================================= +-END OF WebGrease NOTICES, INFORMATION, AND LICENSE +- + %% WiX NOTICES, INFORMATION, AND LICENSE BEGIN HERE + ========================================= + Microsoft Reciprocal License (MS-RL) From 3dc05150cf234f76f6936dcb2853d31a0da1f60e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 07:52:46 -0500 Subject: [PATCH 06/33] [release/8.0] Update azure-activedirectory-identitymodel-extensions-for-dotnet to 7.0.3 (#229) --- .../0001-fix-for-source-build.patch | 30 ++++++++++--------- ...emove-commit-SHA-from-version-string.patch | 4 +-- ...y-identitymodel-extensions-for-dotnet.proj | 2 +- ...ectory-identitymodel-extensions-for-dotnet | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0001-fix-for-source-build.patch b/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0001-fix-for-source-build.patch index 2874b8d70c..e2cd88e7c7 100644 --- a/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0001-fix-for-source-build.patch +++ b/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0001-fix-for-source-build.patch @@ -1,10 +1,10 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Eric Erhardt -Date: Mon, 11 Sep 2023 11:09:12 -0500 +Date: Tue, 17 Oct 2023 12:59:45 -0500 Subject: [PATCH] fix for source build --- - build/common.props | 5 ----- + build/common.props | 6 +----- build/dependencies.props | 8 ++++---- build/targets.props | 4 ++-- buildConfiguration.xml | 2 +- @@ -19,20 +19,22 @@ Subject: [PATCH] fix for source build .../Microsoft.IdentityModel.Validators.csproj | 7 ------- .../Microsoft.IdentityModel.Xml.csproj | 7 ------- .../System.IdentityModel.Tokens.Jwt.csproj | 7 ------- - 15 files changed, 7 insertions(+), 89 deletions(-) + 15 files changed, 8 insertions(+), 89 deletions(-) diff --git a/build/common.props b/build/common.props -index 001d7daa..77c12525 100644 +index 9f99de11..143d8509 100644 --- a/build/common.props +++ b/build/common.props -@@ -1,6 +1,5 @@ - +@@ -35,7 +35,7 @@ + + + +- true ++ false + 7.0.0 + -- - - - -@@ -43,9 +42,5 @@ +@@ -47,9 +47,5 @@ $(NoWarn);SYSLIB0050 $(NoWarn);SYSLIB0051 @@ -80,7 +82,7 @@ index f020b59b..934aae74 100644 diff --git a/buildConfiguration.xml b/buildConfiguration.xml -index 8920a438..1cf387b1 100644 +index 83cd80e1..868c5395 100644 --- a/buildConfiguration.xml +++ b/buildConfiguration.xml @@ -1,7 +1,7 @@ @@ -89,8 +91,8 @@ index 8920a438..1cf387b1 100644 3.5.0-rc-1285 - net461,netstandard2.0 + netstandard2.0 - 7.0.0 - + 7.0.3 + preview diff --git a/src/Microsoft.IdentityModel.Abstractions/Microsoft.IdentityModel.Abstractions.csproj b/src/Microsoft.IdentityModel.Abstractions/Microsoft.IdentityModel.Abstractions.csproj index f7a11afe..4a87af91 100644 diff --git a/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0002-Remove-commit-SHA-from-version-string.patch b/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0002-Remove-commit-SHA-from-version-string.patch index 46681b3b02..1841ecfc94 100644 --- a/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0002-Remove-commit-SHA-from-version-string.patch +++ b/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0002-Remove-commit-SHA-from-version-string.patch @@ -18,8 +18,8 @@ index abbcfac6..2c6a119e 100755 -commitSha=$(git rev-parse HEAD) - - assemblyVersion=$(sed -n 's/.*\([^<]*\)<.*/\1/p' $PWD/buildConfiguration.xml) - assemblyFileVersion="$assemblyVersion.${dateTimeStamp::-6}" # Trim minutes/seconds + assemblyVersion=$(sed -n 's/.*\([^<]*\)<.*/\1/p' ${scriptroot}/buildConfiguration.xml) + assemblyFileVersion="$assemblyVersion.${dateTimeStamp::$((${#dateTimeStamp} - 6))}" # Trim minutes/seconds -assemblyInformationalVersion="$assemblyVersion.$dateTimeStamp.$commitSha" +assemblyInformationalVersion="$assemblyVersion.$dateTimeStamp" diff --git a/repo-projects/azure-activedirectory-identitymodel-extensions-for-dotnet.proj b/repo-projects/azure-activedirectory-identitymodel-extensions-for-dotnet.proj index 72306007d9..229298066e 100644 --- a/repo-projects/azure-activedirectory-identitymodel-extensions-for-dotnet.proj +++ b/repo-projects/azure-activedirectory-identitymodel-extensions-for-dotnet.proj @@ -18,7 +18,7 @@ $(BuildCommandArgs) /p:Configuration=$(Configuration) $(BuildCommandArgs) /v:$(LogVerbosity) $(BuildCommandArgs) $(RedirectRepoOutputToLog) - $(BuildCommandArgs) /p:Version=7.0.0 + $(BuildCommandArgs) /p:Version=7.0.3 $(BuildCommandArgs) --output $(ProjectDirectory)pack diff --git a/src/azure-activedirectory-identitymodel-extensions-for-dotnet b/src/azure-activedirectory-identitymodel-extensions-for-dotnet index bf4cb251a8..bb354ceabe 160000 --- a/src/azure-activedirectory-identitymodel-extensions-for-dotnet +++ b/src/azure-activedirectory-identitymodel-extensions-for-dotnet @@ -1 +1 @@ -Subproject commit bf4cb251a85f1b27bbb208c703f6f3105bdb24ca +Subproject commit bb354ceabed19189245e075abb864f327b6c14ad From 776b128c95ef681d125adc3eb69f12d480418bb5 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 16 Nov 2023 20:25:54 +0000 Subject: [PATCH 07/33] [release/8.0] Update dependencies from dotnet/source-build-reference-packages (#225) [release/8.0] Update dependencies from dotnet/source-build-reference-packages - Merge branch 'release/8.0' into darc-release/8.0-14f9fe7a-52a3-4922-bdb9-b48f80706f2c - Include artifacts built in SBE when considering prebuilts --- eng/SourceBuild.props | 6 ++++++ eng/Version.Details.xml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props index 7855e7141b..609b391564 100644 --- a/eng/SourceBuild.props +++ b/eng/SourceBuild.props @@ -7,4 +7,10 @@ true + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5ac4ace5d1..e992ff252f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 4665b3d04e1da3796b965c3c3e3b97f55c449a6e - + https://github.com/dotnet/source-build-reference-packages - 7b55da982fc6e71c1776c4de89111aee0eecb45a + fa4c0e8f53ef2541a23e519af4dfb86cb88e1bae From e844aa02a05b90d8cbe499676ec6ee0f19ec4980 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:39:19 +0000 Subject: [PATCH 08/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20231115.1 (#236) [release/8.0] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e992ff252f..7e332c607d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 4665b3d04e1da3796b965c3c3e3b97f55c449a6e - + https://github.com/dotnet/source-build-reference-packages - fa4c0e8f53ef2541a23e519af4dfb86cb88e1bae + 95f83e27806330fec09edd96e06bba3acabe3f35 From 6c9557fbf18398d6f226bfd16ce3876f1fe2e940 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 4 Jan 2024 13:19:30 -0600 Subject: [PATCH 09/33] [release/8.0] update docker-creds-provider to b381eafbeecb1039f5839fc98ef45e7b3e52dee9 (a.k.a version 2.2.1) (#247) --- .gitmodules | 7 +++++-- repo-projects/docker-creds-provider-2.2.0.proj | 7 +++++++ repo-projects/docker-creds-provider-2.2.1.proj | 7 +++++++ ...r-creds-provider.proj => docker-creds-provider.targets} | 5 ++++- src/{docker-creds-provider => docker-creds-provider-2.2.0} | 0 src/docker-creds-provider-2.2.1 | 1 + 6 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 repo-projects/docker-creds-provider-2.2.0.proj create mode 100644 repo-projects/docker-creds-provider-2.2.1.proj rename repo-projects/{docker-creds-provider.proj => docker-creds-provider.targets} (94%) rename src/{docker-creds-provider => docker-creds-provider-2.2.0} (100%) create mode 160000 src/docker-creds-provider-2.2.1 diff --git a/.gitmodules b/.gitmodules index 19152635c4..210cff8930 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,8 +16,11 @@ [submodule "src/azure-activedirectory-identitymodel-extensions-for-dotnet"] path = src/azure-activedirectory-identitymodel-extensions-for-dotnet url = https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet.git -[submodule "src/docker-creds-provider"] - path = src/docker-creds-provider +[submodule "src/docker-creds-provider-2.2.0"] + path = src/docker-creds-provider-2.2.0 + url = https://github.com/mthalman/docker-creds-provider +[submodule "src/docker-creds-provider-2.2.1"] + path = src/docker-creds-provider-2.2.1 url = https://github.com/mthalman/docker-creds-provider [submodule "xunit"] path = src/xunit diff --git a/repo-projects/docker-creds-provider-2.2.0.proj b/repo-projects/docker-creds-provider-2.2.0.proj new file mode 100644 index 0000000000..75f1ca5415 --- /dev/null +++ b/repo-projects/docker-creds-provider-2.2.0.proj @@ -0,0 +1,7 @@ + + + 2.2.0 + + + + diff --git a/repo-projects/docker-creds-provider-2.2.1.proj b/repo-projects/docker-creds-provider-2.2.1.proj new file mode 100644 index 0000000000..5e8c00c009 --- /dev/null +++ b/repo-projects/docker-creds-provider-2.2.1.proj @@ -0,0 +1,7 @@ + + + 2.2.1 + + + + diff --git a/repo-projects/docker-creds-provider.proj b/repo-projects/docker-creds-provider.targets similarity index 94% rename from repo-projects/docker-creds-provider.proj rename to repo-projects/docker-creds-provider.targets index 049b30ec33..585991996f 100644 --- a/repo-projects/docker-creds-provider.proj +++ b/repo-projects/docker-creds-provider.targets @@ -1,9 +1,12 @@ + + $(ProjectDirectory)/src/Valleysoft.DockerCredsProvider/bin/$(Configuration)/ - 2.2.0 diff --git a/src/docker-creds-provider b/src/docker-creds-provider-2.2.0 similarity index 100% rename from src/docker-creds-provider rename to src/docker-creds-provider-2.2.0 diff --git a/src/docker-creds-provider-2.2.1 b/src/docker-creds-provider-2.2.1 new file mode 160000 index 0000000000..b381eafbee --- /dev/null +++ b/src/docker-creds-provider-2.2.1 @@ -0,0 +1 @@ +Subproject commit b381eafbeecb1039f5839fc98ef45e7b3e52dee9 From 7134e53b6b1210a1ce8838b12b8f6071e0a3433b Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Tue, 9 Jan 2024 16:13:41 -0600 Subject: [PATCH 10/33] Update to IdentityModel 7.1.2 (#251) --- .../0001-fix-for-source-build.patch | 2 +- ...ure-activedirectory-identitymodel-extensions-for-dotnet.proj | 2 +- src/azure-activedirectory-identitymodel-extensions-for-dotnet | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0001-fix-for-source-build.patch b/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0001-fix-for-source-build.patch index e2cd88e7c7..41bf255a6b 100644 --- a/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0001-fix-for-source-build.patch +++ b/patches/azure-activedirectory-identitymodel-extensions-for-dotnet/0001-fix-for-source-build.patch @@ -91,7 +91,7 @@ index 83cd80e1..868c5395 100644 3.5.0-rc-1285 - net461,netstandard2.0 + netstandard2.0 - 7.0.3 + 7.1.2 preview diff --git a/src/Microsoft.IdentityModel.Abstractions/Microsoft.IdentityModel.Abstractions.csproj b/src/Microsoft.IdentityModel.Abstractions/Microsoft.IdentityModel.Abstractions.csproj diff --git a/repo-projects/azure-activedirectory-identitymodel-extensions-for-dotnet.proj b/repo-projects/azure-activedirectory-identitymodel-extensions-for-dotnet.proj index 229298066e..c63568adf9 100644 --- a/repo-projects/azure-activedirectory-identitymodel-extensions-for-dotnet.proj +++ b/repo-projects/azure-activedirectory-identitymodel-extensions-for-dotnet.proj @@ -18,7 +18,7 @@ $(BuildCommandArgs) /p:Configuration=$(Configuration) $(BuildCommandArgs) /v:$(LogVerbosity) $(BuildCommandArgs) $(RedirectRepoOutputToLog) - $(BuildCommandArgs) /p:Version=7.0.3 + $(BuildCommandArgs) /p:Version=7.1.2 $(BuildCommandArgs) --output $(ProjectDirectory)pack diff --git a/src/azure-activedirectory-identitymodel-extensions-for-dotnet b/src/azure-activedirectory-identitymodel-extensions-for-dotnet index bb354ceabe..a607fa5e00 160000 --- a/src/azure-activedirectory-identitymodel-extensions-for-dotnet +++ b/src/azure-activedirectory-identitymodel-extensions-for-dotnet @@ -1 +1 @@ -Subproject commit bb354ceabed19189245e075abb864f327b6c14ad +Subproject commit a607fa5e0005a6178cf1d2fed4fa0f8179cdb186 From 83274d94c7e2ff21081b0d75ecbec2da2241f831 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:30:22 +0000 Subject: [PATCH 11/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240111.1 (#255) [release/8.0] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7e332c607d..d243bd81ae 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 4665b3d04e1da3796b965c3c3e3b97f55c449a6e - + https://github.com/dotnet/source-build-reference-packages - 95f83e27806330fec09edd96e06bba3acabe3f35 + 453a37ef7ae6c335cd49b3b9ab7713c87faeb265 From 4616461a017409a7f7c17033a7115ff98c998ca9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 8 Mar 2024 15:48:38 -0600 Subject: [PATCH 12/33] [release/8.0] Migrate to 1ES pipeline templates (#277) --- .vsts.pipelines/builds/ci-public.yml | 52 +++++++ .vsts.pipelines/builds/ci.yml | 102 +++++++------ .vsts.pipelines/guardian/.gdnsuppress | 207 ++++++++++++++++++++++++++ 3 files changed, 316 insertions(+), 45 deletions(-) create mode 100644 .vsts.pipelines/builds/ci-public.yml create mode 100644 .vsts.pipelines/guardian/.gdnsuppress diff --git a/.vsts.pipelines/builds/ci-public.yml b/.vsts.pipelines/builds/ci-public.yml new file mode 100644 index 0000000000..f4573ab128 --- /dev/null +++ b/.vsts.pipelines/builds/ci-public.yml @@ -0,0 +1,52 @@ +trigger: + batch: true + branches: + include: + - main + - release/* + paths: + exclude: + - '*.md' + +pr: + branches: + include: + - main + - release/* + paths: + exclude: + - '*.md' + +variables: + - template: /eng/common/templates/variables/pool-providers.yml + - name: Codeql.Enable + value: true + +stages: +- stage: build + displayName: Build + jobs: + - template: /eng/common/templates/jobs/jobs.yml + parameters: + enablePublishUsingPipelines: true + enablePublishBuildAssets: true + artifacts: + publish: + artifacts: true + manifests: true + jobs: + - job: SourceBuild_Managed + displayName: Source-Build (Managed) + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + workspace: + clean: all + variables: + - name: _BuildConfig + value: Release + steps: + - checkout: self + submodules: recursive + - template: /eng/common/templates/steps/source-build.yml diff --git a/.vsts.pipelines/builds/ci.yml b/.vsts.pipelines/builds/ci.yml index b979ffaaf6..49bf7810e8 100644 --- a/.vsts.pipelines/builds/ci.yml +++ b/.vsts.pipelines/builds/ci.yml @@ -7,7 +7,6 @@ trigger: paths: exclude: - '*.md' - pr: branches: include: @@ -16,49 +15,62 @@ pr: paths: exclude: - '*.md' - variables: - - name: Codeql.Enable - value: true - -stages: -- stage: build - displayName: Build - jobs: - - template: /eng/common/templates/jobs/jobs.yml - parameters: - enablePublishUsingPipelines: true - enablePublishBuildAssets: true - artifacts: - publish: - artifacts: true - manifests: true +- template: /eng/common/templates-official/variables/pool-providers.yml +- name: TeamName + value: DotNetSourceBuild +- name: Codeql.Enable + value: true +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + suppression: + suppressionFile: $(Build.SourcesDirectory)\.vsts.pipelines\guardian\.gdnsuppress + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: build + displayName: Build jobs: - - - job: SourceBuild_Managed - displayName: Source-Build (Managed) - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' - workspace: - clean: all - variables: - - name: _BuildConfig - value: Release - steps: - - checkout: self - submodules: recursive - - template: /eng/common/templates/steps/source-build.yml - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/post-build/post-build.yml - parameters: - publishingInfraVersion: 3 - enableSourceLinkValidation: true - enableSigningValidation: false - + - template: /eng/common/templates-official/jobs/jobs.yml@self + parameters: + enablePublishUsingPipelines: true + enablePublishBuildAssets: true + artifacts: + publish: + artifacts: true + manifests: true + jobs: + - job: SourceBuild_Managed + displayName: Source-Build (Managed) + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204-pt + os: linux + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + workspace: + clean: all + variables: + - name: _BuildConfig + value: Release + steps: + - checkout: self + submodules: recursive + - template: /eng/common/templates-official/steps/source-build.yml + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/templates-official/post-build/post-build.yml@self + parameters: + publishingInfraVersion: 3 + enableSourceLinkValidation: true + enableSigningValidation: false diff --git a/.vsts.pipelines/guardian/.gdnsuppress b/.vsts.pipelines/guardian/.gdnsuppress new file mode 100644 index 0000000000..be3f6fe9bb --- /dev/null +++ b/.vsts.pipelines/guardian/.gdnsuppress @@ -0,0 +1,207 @@ +{ + "hydrated": false, + "properties": { + "helpUri": "https://eng.ms/docs/microsoft-security/security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/microsoft-guardian/general/suppressions", + "hydrationStatus": "This file does not contain identifying data. It is safe to check into your repo. To hydrate this file with identifying data, run `guardian hydrate --help` and follow the guidance." + }, + "version": "1.0.0", + "suppressionSets": { + "default": { + "name": "default", + "createdDate": "2024-03-07 17:09:14Z", + "lastUpdatedDate": "2024-03-07 17:09:14Z" + } + }, + "results": { + "31128318971be3d77cbd3aaf7b6a06d65b1874334a143ee500c7fccb5aa89427": { + "signature": "31128318971be3d77cbd3aaf7b6a06d65b1874334a143ee500c7fccb5aa89427", + "alternativeSignatures": [ + "9106dc3b9a335702dc4feeeed54285f07d8a06494f38fc23167f6158793928dc" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "992b26983b997813a410dfc25048f3b218c6fc02fc14a5c2ad431ec8e022ac79": { + "signature": "992b26983b997813a410dfc25048f3b218c6fc02fc14a5c2ad431ec8e022ac79", + "alternativeSignatures": [ + "23e97da32b7142c282727c96d07fd5ce6aefd6ef26f02e91cb471eb7863542f8" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "53b10a5fb6059b0b229ad32c6278123a5603386f65d9e1c5684a2333f2e1dc62": { + "signature": "53b10a5fb6059b0b229ad32c6278123a5603386f65d9e1c5684a2333f2e1dc62", + "alternativeSignatures": [ + "cd7b0b0937cfa32a98962a528bd99ede0181ae41a609df430f35fd30763166c4" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "826c1d41ea210af2fc3a7ff120c606aecee5bede4bba54c10512a998f53fafd0": { + "signature": "826c1d41ea210af2fc3a7ff120c606aecee5bede4bba54c10512a998f53fafd0", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "2123572b79400b0335c85250c4aff8f6142cea7a437a37153904f79ddafa64fc": { + "signature": "2123572b79400b0335c85250c4aff8f6142cea7a437a37153904f79ddafa64fc", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "3eced103c90fbc6d04eb6af47fd6213563948693acfc44c884d4feef4f4c4900": { + "signature": "3eced103c90fbc6d04eb6af47fd6213563948693acfc44c884d4feef4f4c4900", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "1ef98823231057834e7ee94c5b5b8316a736ab1a66bce45df0455867c7562fe9": { + "signature": "1ef98823231057834e7ee94c5b5b8316a736ab1a66bce45df0455867c7562fe9", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "fbcf8ea13d120926f0777f7f73d318d9ef6f18c9e88b844993e1bc7ba7cfc79c": { + "signature": "fbcf8ea13d120926f0777f7f73d318d9ef6f18c9e88b844993e1bc7ba7cfc79c", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "84d9c760946210c88b698e6084f5db7a33353ecef49fa9dfaf424b545014980b": { + "signature": "84d9c760946210c88b698e6084f5db7a33353ecef49fa9dfaf424b545014980b", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "a0bb972ab1a203ccb881e90b3593b65127f78d0dc007c5ca83e1e6d07338e8c3": { + "signature": "a0bb972ab1a203ccb881e90b3593b65127f78d0dc007c5ca83e1e6d07338e8c3", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "7363daf29c8c84a8ed6991e3c7fc3eb44115b0403b6b47fac99a9aa2cd3b1998": { + "signature": "7363daf29c8c84a8ed6991e3c7fc3eb44115b0403b6b47fac99a9aa2cd3b1998", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "a9485af66c8f757c02c53aa5f7cf02cde5c1c9e60e8c70768ff3a124486fa3a7": { + "signature": "a9485af66c8f757c02c53aa5f7cf02cde5c1c9e60e8c70768ff3a124486fa3a7", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "d95887551b5f4f802485e351daca216de95c8a328bab614ddefc518b1acfed26": { + "signature": "d95887551b5f4f802485e351daca216de95c8a328bab614ddefc518b1acfed26", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "9cfa6b8f64d95cf2469afcef66aae5e9c76c64a671b082ff42d00ce42c434aff": { + "signature": "9cfa6b8f64d95cf2469afcef66aae5e9c76c64a671b082ff42d00ce42c434aff", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "9c75c0e9d0e58ff6d39b74a0d8326609b8465c59ac2b03eccc7eaf4f50ae32d1": { + "signature": "9c75c0e9d0e58ff6d39b74a0d8326609b8465c59ac2b03eccc7eaf4f50ae32d1", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "74856bc4ed4c67c9c0c6c69c7e8a3a01ffcf7f9a89d332ca68a0e6ecb51ad511": { + "signature": "74856bc4ed4c67c9c0c6c69c7e8a3a01ffcf7f9a89d332ca68a0e6ecb51ad511", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "7318b6ff65864c4b5cc05a2439ffc3b7be06114f2a7007da6253fed91d1c9e56": { + "signature": "7318b6ff65864c4b5cc05a2439ffc3b7be06114f2a7007da6253fed91d1c9e56", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "b35cc2f8fe7a2aba483d0f29517153876385314dffb249548530d2e5463bdf93": { + "signature": "b35cc2f8fe7a2aba483d0f29517153876385314dffb249548530d2e5463bdf93", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "8d3d289c461dd6454c98dbcaa5fef608dae8a32c68929cffc6a9db8b2826b840": { + "signature": "8d3d289c461dd6454c98dbcaa5fef608dae8a32c68929cffc6a9db8b2826b840", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "6bb3b72839c43d3ff9a6aba1e5cc5708f01865416f020d1a26643cffff8fa74f": { + "signature": "6bb3b72839c43d3ff9a6aba1e5cc5708f01865416f020d1a26643cffff8fa74f", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "2610d6745224ba98106d22e2c66c4abce8ffafecb57f444ed8a0f2a542fa0538": { + "signature": "2610d6745224ba98106d22e2c66c4abce8ffafecb57f444ed8a0f2a542fa0538", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "40d2ad8a369cd2c1f07465a2a68b6dc202f5ea371c2576f2b0cc18c82f3aa675": { + "signature": "40d2ad8a369cd2c1f07465a2a68b6dc202f5ea371c2576f2b0cc18c82f3aa675", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + }, + "affacdd646b09264bc94c75ff52aafe58f0cd1f2c09cb50033014c38c4823d86": { + "signature": "affacdd646b09264bc94c75ff52aafe58f0cd1f2c09cb50033014c38c4823d86", + "alternativeSignatures": [], + "memberOf": [ + "default" + ], + "createdDate": "2024-03-07 17:09:14Z" + } + } +} \ No newline at end of file From 7a9b99e457a2b9792a3c17ccaf95d80038725108 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 8 Mar 2024 16:20:06 -0600 Subject: [PATCH 13/33] Update dependencies from https://github.com/dotnet/arcade build 20240306.1 (#278) Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.23451.1 -> To Version 8.0.0-beta.24156.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/darc-init.ps1 | 2 +- eng/common/darc-init.sh | 2 +- .../post-build/add-build-to-channel.ps1 | 2 +- eng/common/post-build/publish-using-darc.ps1 | 6 +- .../post-build/trigger-subscriptions.ps1 | 2 +- eng/common/sdk-task.ps1 | 2 +- eng/common/templates-official/job/job.yml | 255 ++++++++++++++++ .../templates-official/job/onelocbuild.yml | 112 +++++++ .../job/publish-build-assets.yml | 153 ++++++++++ .../templates-official/job/source-build.yml | 67 ++++ .../job/source-index-stage1.yml | 68 +++++ .../templates-official/jobs/codeql-build.yml | 31 ++ eng/common/templates-official/jobs/jobs.yml | 97 ++++++ .../templates-official/jobs/source-build.yml | 46 +++ .../post-build/common-variables.yml | 22 ++ .../post-build/post-build.yml | 285 ++++++++++++++++++ .../post-build/setup-maestro-vars.yml | 70 +++++ .../post-build/trigger-subscription.yml | 13 + .../steps/add-build-to-channel.yml | 13 + .../templates-official/steps/build-reason.yml | 12 + .../steps/component-governance.yml | 13 + .../steps/execute-codeql.yml | 32 ++ .../templates-official/steps/execute-sdl.yml | 88 ++++++ .../steps/generate-sbom.yml | 48 +++ .../templates-official/steps/publish-logs.yml | 23 ++ .../templates-official/steps/retain-build.yml | 28 ++ .../steps/send-to-helix.yml | 91 ++++++ .../templates-official/steps/source-build.yml | 129 ++++++++ .../variables/pool-providers.yml | 45 +++ .../variables/sdl-variables.yml | 7 + eng/common/templates/job/job.yml | 2 +- .../templates/job/publish-build-assets.yml | 18 +- .../templates/post-build/common-variables.yml | 2 +- .../templates/post-build/post-build.yml | 20 +- eng/common/templates/steps/generate-sbom.yml | 2 +- .../templates/variables/pool-providers.yml | 12 +- eng/common/tools.ps1 | 16 +- eng/common/tools.sh | 7 +- global.json | 4 +- 40 files changed, 1806 insertions(+), 45 deletions(-) create mode 100644 eng/common/templates-official/job/job.yml create mode 100644 eng/common/templates-official/job/onelocbuild.yml create mode 100644 eng/common/templates-official/job/publish-build-assets.yml create mode 100644 eng/common/templates-official/job/source-build.yml create mode 100644 eng/common/templates-official/job/source-index-stage1.yml create mode 100644 eng/common/templates-official/jobs/codeql-build.yml create mode 100644 eng/common/templates-official/jobs/jobs.yml create mode 100644 eng/common/templates-official/jobs/source-build.yml create mode 100644 eng/common/templates-official/post-build/common-variables.yml create mode 100644 eng/common/templates-official/post-build/post-build.yml create mode 100644 eng/common/templates-official/post-build/setup-maestro-vars.yml create mode 100644 eng/common/templates-official/post-build/trigger-subscription.yml create mode 100644 eng/common/templates-official/steps/add-build-to-channel.yml create mode 100644 eng/common/templates-official/steps/build-reason.yml create mode 100644 eng/common/templates-official/steps/component-governance.yml create mode 100644 eng/common/templates-official/steps/execute-codeql.yml create mode 100644 eng/common/templates-official/steps/execute-sdl.yml create mode 100644 eng/common/templates-official/steps/generate-sbom.yml create mode 100644 eng/common/templates-official/steps/publish-logs.yml create mode 100644 eng/common/templates-official/steps/retain-build.yml create mode 100644 eng/common/templates-official/steps/send-to-helix.yml create mode 100644 eng/common/templates-official/steps/source-build.yml create mode 100644 eng/common/templates-official/variables/pool-providers.yml create mode 100644 eng/common/templates-official/variables/sdl-variables.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d243bd81ae..f2aad04ada 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 4665b3d04e1da3796b965c3c3e3b97f55c449a6e + 1307d3e675219bf384f17764651f46767a07960b diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 435e764134..8fda30bdce 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,6 +1,6 @@ param ( $darcVersion = $null, - $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16', + $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16', $verbosity = 'minimal', $toolpath = $null ) diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 84c1d0cc2e..c305ae6bd7 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -2,7 +2,7 @@ source="${BASH_SOURCE[0]}" darcVersion='' -versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16' +versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16' verbosity='minimal' while [[ $# > 0 ]]; do diff --git a/eng/common/post-build/add-build-to-channel.ps1 b/eng/common/post-build/add-build-to-channel.ps1 index de2d957922..49938f0c89 100644 --- a/eng/common/post-build/add-build-to-channel.ps1 +++ b/eng/common/post-build/add-build-to-channel.ps1 @@ -2,7 +2,7 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' ) diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 8508397d77..5a3a32ea8d 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -3,7 +3,7 @@ param( [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, [Parameter(Mandatory=$true)][string] $AzdoToken, [Parameter(Mandatory=$true)][string] $MaestroToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters @@ -12,7 +12,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $darc = Get-Darc + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() @@ -46,7 +46,7 @@ try { } Write-Host 'done.' -} +} catch { Write-Host $_ Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 index 55dea518ac..ac9a95778f 100644 --- a/eng/common/post-build/trigger-subscriptions.ps1 +++ b/eng/common/post-build/trigger-subscriptions.ps1 @@ -2,7 +2,7 @@ param( [Parameter(Mandatory=$true)][string] $SourceRepo, [Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' ) diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 6c4ac6fec1..73828dd30d 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.6.0-2" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 0000000000..9e7bebe9af --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,255 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - publish: artifacts/log + artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: Publish logs + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration + artifact: BuildConfiguration + displayName: Publish build retry configuration + continueOnError: true diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 0000000000..ba9ba49303 --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,112 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 0000000000..ea5104625f --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,153 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 0000000000..8aba3b44bb --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,67 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2-pt + os: linux + + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 0000000000..4b63373917 --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,68 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET Core SDK 6 + inputs: + packageType: sdk + version: 6.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 0000000000..b68d3c2f31 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 0000000000..857a0f8ba4 --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,97 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 0000000000..08e5db9bb1 --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 0000000000..c24193acfc --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,22 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 0000000000..5c98fe1c0f --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,285 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 0000000000..0c87f149a4 --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 0000000000..da669030da --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 0000000000..f67a210d62 --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/build-reason.yml b/eng/common/templates-official/steps/build-reason.yml new file mode 100644 index 0000000000..eba58109b5 --- /dev/null +++ b/eng/common/templates-official/steps/build-reason.yml @@ -0,0 +1,12 @@ +# build-reason.yml +# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons +# to include steps (',' separated). +parameters: + conditions: '' + steps: [] + +steps: + - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: + - ${{ parameters.steps }} + - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 0000000000..0ecec47b0c --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml new file mode 100644 index 0000000000..9b4a5ffa30 --- /dev/null +++ b/eng/common/templates-official/steps/execute-codeql.yml @@ -0,0 +1,32 @@ +parameters: + # Language that should be analyzed. Defaults to csharp + language: csharp + # Build Commands + buildCommands: '' + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + # optional: determines whether to continue the build if the step errors; + sdlContinueOnError: false + +steps: +- template: /eng/common/templates-official/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: '${{ parameters.additionalParameters }} + -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml new file mode 100644 index 0000000000..07426fde05 --- /dev/null +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -0,0 +1,88 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: GuardianConnect + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian (Overridden) + +- ${{ if eq(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL (Overridden) + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianCliLocation $(GuardianCliLocation) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() + + # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration + # with the "SARIF SAST Scans Tab" Azure DevOps extension + - task: CopyFiles@2 + displayName: Copy SARIF files + inputs: + flattenFolders: true + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + contents: '**/*.sarif' + targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs + condition: succeededOrFailed() + + # Use PublishBuildArtifacts because the SARIF extension only checks this case + # see microsoft/sarif-azuredevops-extension#4 + - task: PublishBuildArtifacts@1 + displayName: Publish SARIF files to CodeAnalysisLogs container + inputs: + pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 0000000000..1bf43bf807 --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,48 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 8.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 0000000000..04012fed18 --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,23 @@ +parameters: + StageLabel: '' + JobLabel: '' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 0000000000..83d97a26a0 --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 0000000000..3eb7e2d5f8 --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,91 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 0000000000..829f17c34d --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,129 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 0000000000..beab7d1bfb --- /dev/null +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022-pt + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 0000000000..dbdd66d4a4 --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e20ee3a983..e24ca2f46f 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -136,7 +136,7 @@ jobs: condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - task: DownloadPipelineArtifact@2 diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 42017109f3..8ec0151def 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -58,7 +58,7 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: @@ -71,8 +71,8 @@ jobs: checkDownloadedFiles: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@0 + + - task: NuGetAuthenticate@1 - task: PowerShell@2 displayName: Publish Build Assets @@ -81,12 +81,12 @@ jobs: arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -95,7 +95,7 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - + - task: PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: @@ -121,7 +121,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') + condition: eq(variables['SymbolExclusionFile'], 'true') inputs: PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' PublishLocation: Container @@ -137,7 +137,7 @@ jobs: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' @@ -148,4 +148,4 @@ jobs: - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/templates/steps/publish-logs.yml parameters: - JobLabel: 'Publish_Artifacts_Logs' + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index c24193acfc..173914f236 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -7,7 +7,7 @@ variables: # Default Maestro++ API Endpoint and API Version - name: MaestroApiEndPoint - value: "https://maestro-prod.westus2.cloudapp.azure.com" + value: "https://maestro.dot.net" - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index ef720f9d78..aba44a25a3 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -39,7 +39,7 @@ parameters: displayName: Enable NuGet validation type: boolean default: true - + - name: publishInstallersAndChecksums displayName: Publish installers and checksums type: boolean @@ -131,8 +131,8 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - job: displayName: Signing Validation @@ -169,7 +169,7 @@ stages: # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 displayName: 'Authenticate to AzDO Feeds' # Signing validation will optionally work with the buildmanifest file which is downloaded from @@ -221,9 +221,9 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) -GHCommit $(Build.SourceVersion) -SourcelinkCliVersion $(SourceLinkCLIVersion) continueOnError: true @@ -258,7 +258,7 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml @@ -266,13 +266,13 @@ stages: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 - task: PowerShell@2 displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38f..2b21eae427 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -5,7 +5,7 @@ # IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: - PackageVersion: 7.0.0 + PackageVersion: 8.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 9cc5c550d3..d236f9fdbb 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -1,15 +1,15 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, # otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. -# Motivation: +# Motivation: # Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS # (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing # (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template # file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. -# How to use: +# How to use: # This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). # If we find alternate naming conventions in broad usage it can be added to the condition below. # @@ -54,4 +54,4 @@ variables: False, 'NetCore1ESPool-Internal' ) - ] \ No newline at end of file + ] diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index aa74ab4a81..eb188cfda4 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -379,13 +379,13 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '17.6' + $vsMinVersionReqdStr = '17.7' $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.6.0-2 - $defaultXCopyMSBuildVersion = '17.6.0-2' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2 + $defaultXCopyMSBuildVersion = '17.8.1-2' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -601,7 +601,15 @@ function InitializeBuildTool() { ExitWithExitCode 1 } $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net8.0' } + + # Use override if it exists - commonly set by source-build + if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) { + $initializeBuildToolFramework="net8.0" + } else { + $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework + } + + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e8d4789433..3392e3a999 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -341,7 +341,12 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" - _InitializeBuildToolFramework="net8.0" + # use override if it exists - commonly set by source-build + if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then + _InitializeBuildToolFramework="net8.0" + else + _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}" + fi } # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 diff --git a/global.json b/global.json index 987bf09dc3..cdee645953 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "8.0.100-preview.7.23376.3" + "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23451.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24156.1" } } From 00fb7841c80b44262646e57bcfbe90a1b7bc3151 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 13:35:30 +0000 Subject: [PATCH 14/33] Update dependencies from https://github.com/dotnet/arcade build 20240308.4 (#285) [release/8.0] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/SetupNugetSources.ps1 | 26 +++++++++++------------ eng/common/templates-official/job/job.yml | 4 ++++ eng/common/templates/job/job.yml | 4 ++++ global.json | 2 +- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f2aad04ada..1ec29fc30e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 1307d3e675219bf384f17764651f46767a07960b + 052a4b9e7a9bdb9744c86c05665f1b46e4d59b15 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6c65e81925..efa2fd72bf 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,12 +48,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +81,18 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } -$doc.Save($filename) +$doc.Save($filename) \ No newline at end of file diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 9e7bebe9af..647e3f92e5 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e24ca2f46f..8ec5c4f2d9 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/global.json b/global.json index cdee645953..ff6fd14602 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24156.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24158.4" } } From 629dfe34c35a9399849a85c591f8794d0ecd373c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 13:03:55 +0000 Subject: [PATCH 15/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build (#290) [release/8.0] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1ec29fc30e..f259a71c35 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 052a4b9e7a9bdb9744c86c05665f1b46e4d59b15 - + https://github.com/dotnet/source-build-reference-packages - 453a37ef7ae6c335cd49b3b9ab7713c87faeb265 + 79827eed138fd2575a8b24820b4f385ee4ffb6e6 From 0fac378047750fa8bd850a98b159560f9f7627c3 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 13:19:59 +0000 Subject: [PATCH 16/33] Update dependencies from https://github.com/dotnet/arcade build (#292) [release/8.0] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/templates-official/job/job.yml | 18 +++++++++++------- .../job/publish-build-assets.yml | 10 ++++++---- global.json | 2 +- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f259a71c35..9ead692ce0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 052a4b9e7a9bdb9744c86c05665f1b46e4d59b15 + f311667e0587f19c3fa9553a909975662107a351 diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 647e3f92e5..a2709d1056 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -206,9 +206,11 @@ jobs: continueOnError: true condition: always() - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - publish: artifacts/log - artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: Publish logs + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: 'artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' continueOnError: true condition: always() @@ -253,7 +255,9 @@ jobs: IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration - artifact: BuildConfiguration - displayName: Publish build retry configuration - continueOnError: true + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index ea5104625f..53138622fe 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -94,14 +94,16 @@ jobs: inputs: targetType: inline script: | - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) - task: 1ES.PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' PublishLocation: Container ArtifactName: ReleaseConfigs diff --git a/global.json b/global.json index ff6fd14602..d719bb08ca 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24158.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24165.4" } } From 300e99190e6ae1983681694dbdd5f75f0c692081 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:46:34 +0000 Subject: [PATCH 17/33] Update dependencies from https://github.com/dotnet/arcade build (#299) [release/8.0] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/templates-official/job/job.yml | 2 +- global.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9ead692ce0..fbfcff507c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - f311667e0587f19c3fa9553a909975662107a351 + ceb071c1060b8e6de404c065b4045442570caa18 diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index a2709d1056..0604277a2f 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -128,7 +128,7 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@3 + - task: MicroBuildSigningPlugin@4 displayName: Install MicroBuild plugin inputs: signType: $(_SignType) diff --git a/global.json b/global.json index d719bb08ca..372f36d934 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24165.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24172.5" } } From bfcb59fce0611ec97723d613f8ccfeb618f8519c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 19:43:32 +0000 Subject: [PATCH 18/33] Update dependencies from https://github.com/dotnet/arcade build 20240404.3 (#306) [release/8.0] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/native/init-compiler.sh | 2 +- eng/common/templates-official/job/job.yml | 1 + eng/common/templates-official/job/onelocbuild.yml | 2 +- .../templates-official/job/publish-build-assets.yml | 4 ++-- eng/common/templates-official/job/source-build.yml | 2 +- .../templates-official/job/source-index-stage1.yml | 2 +- .../templates-official/post-build/post-build.yml | 10 +++++----- .../templates-official/steps/component-governance.yml | 2 +- .../templates-official/variables/pool-providers.yml | 2 +- eng/common/templates/steps/component-governance.yml | 2 +- global.json | 2 +- 12 files changed, 18 insertions(+), 17 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index fbfcff507c..06b05279ad 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - ceb071c1060b8e6de404c065b4045442570caa18 + 188340e12c0a372b1681ad6a5e72c608021efdba diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index f5c1ec7eaf..2d5660642b 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then # Set default versions if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" + if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 0604277a2f..1f035fee73 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -136,6 +136,7 @@ jobs: feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json env: TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml index ba9ba49303..52b4d05d3f 100644 --- a/eng/common/templates-official/job/onelocbuild.yml +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -56,7 +56,7 @@ jobs: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 53138622fe..589ac80a18 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -60,8 +60,8 @@ jobs: os: windows # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 os: windows steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index 8aba3b44bb..f193dfbe23 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -52,7 +52,7 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2-pt + image: 1es-mariner-2 os: linux ${{ if ne(parameters.platform.pool, '') }}: diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 4b63373917..f0513aee5b 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -33,7 +33,7 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: windows.vs2022.amd64 os: windows steps: diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index 5c98fe1c0f..da1f40958b 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -110,7 +110,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: @@ -150,7 +150,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: - template: setup-maestro-vars.yml @@ -208,7 +208,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: - template: setup-maestro-vars.yml @@ -261,8 +261,8 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 os: windows steps: - template: setup-maestro-vars.yml diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml index 0ecec47b0c..cbba059670 100644 --- a/eng/common/templates-official/steps/component-governance.yml +++ b/eng/common/templates-official/steps/component-governance.yml @@ -4,7 +4,7 @@ parameters: steps: - ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" displayName: Set skipComponentGovernanceDetection variable - ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - task: ComponentGovernanceComponentDetection@0 diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml index beab7d1bfb..1f308b24ef 100644 --- a/eng/common/templates-official/variables/pool-providers.yml +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022-pt +# image: 1es-windows-2022 variables: # Coalesce the target and source branches so we know when a PR targets a release branch diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml index 0ecec47b0c..cbba059670 100644 --- a/eng/common/templates/steps/component-governance.yml +++ b/eng/common/templates/steps/component-governance.yml @@ -4,7 +4,7 @@ parameters: steps: - ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" displayName: Set skipComponentGovernanceDetection variable - ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - task: ComponentGovernanceComponentDetection@0 diff --git a/global.json b/global.json index 372f36d934..67b7f26bc1 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24172.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24204.3" } } From 908177a58a41532b3302c17f1e1a8cf1c1234545 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:57:43 -0500 Subject: [PATCH 19/33] Migrate off deprecated "pt" pool image (#309) Co-authored-by: Matt Thalman --- .vsts.pipelines/builds/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vsts.pipelines/builds/ci.yml b/.vsts.pipelines/builds/ci.yml index 49bf7810e8..eeedff7637 100644 --- a/.vsts.pipelines/builds/ci.yml +++ b/.vsts.pipelines/builds/ci.yml @@ -33,7 +33,7 @@ extends: sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows suppression: suppressionFile: $(Build.SourcesDirectory)\.vsts.pipelines\guardian\.gdnsuppress @@ -56,7 +56,7 @@ extends: displayName: Source-Build (Managed) pool: name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204-pt + image: 1es-ubuntu-2204 os: linux container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' workspace: From 2b7510ccda2be01e2a2b48598498dca24fb69c3a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 13:18:15 +0000 Subject: [PATCH 20/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240501.1 (#316) [release/8.0] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 06b05279ad..c1d079f5fa 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/source-build-reference-packages - 79827eed138fd2575a8b24820b4f385ee4ffb6e6 + 6f814daa935e08b578b1c0c65a1f26ea3317f517 From a3021ef9ed72d7bdf799092a47d2d024fc13bfcd Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 13:16:03 +0000 Subject: [PATCH 21/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240507.2 (#319) [release/8.0] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c1d079f5fa..dd947e2241 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/source-build-reference-packages - 6f814daa935e08b578b1c0c65a1f26ea3317f517 + 6ed73280a6d70f7e7ac39c86f2abe8c10983f0bb From 4f2151df120194f0268944f1b723c14820738fc8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 05:13:36 +0000 Subject: [PATCH 22/33] Update dependencies from https://github.com/dotnet/arcade build 20240516.3 (#320) [release/8.0] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- .../job/source-index-stage1.yml | 49 +++++++++++++------ .../templates/job/source-index-stage1.yml | 44 ++++++++++++----- global.json | 2 +- 4 files changed, 67 insertions(+), 32 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dd947e2241..0ef64be84b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index f0513aee5b..43ee0c202f 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -14,15 +15,15 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - - template: /eng/common/templates-official/variables/pool-providers.yml + - template: /eng/common/templates/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} @@ -33,24 +34,23 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 - os: windows + demands: ImageOverride -equals windows.vs2019.amd64 steps: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -62,7 +62,24 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + + - script: | + echo "Client ID: $(ARM_CLIENT_ID)" + echo "ID Token: $(ARM_ID_TOKEN)" + echo "Tenant ID: $(ARM_TENANT_ID)" + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + displayName: Upload stage1 artifacts to source index \ No newline at end of file diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b98202aa02..43ee0c202f 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -14,14 +15,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - template: /eng/common/templates/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: @@ -40,16 +41,16 @@ jobs: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -61,7 +62,24 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + + - script: | + echo "Client ID: $(ARM_CLIENT_ID)" + echo "ID Token: $(ARM_ID_TOKEN)" + echo "Tenant ID: $(ARM_TENANT_ID)" + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + displayName: Upload stage1 artifacts to source index \ No newline at end of file diff --git a/global.json b/global.json index 67b7f26bc1..63925fe917 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24204.3" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24266.3" } } From b086ae0e4cf1ae64f1237f976fa65221f87cd4a0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 11 Jul 2024 17:01:45 +0000 Subject: [PATCH 23/33] Update dependencies from https://github.com/dotnet/arcade build 20240710.5 (#334) [release/8.0] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/post-build/publish-using-darc.ps1 | 15 +++---- .../job/publish-build-assets.yml | 21 +++++---- .../templates-official/job/source-build.yml | 8 ++++ .../job/source-index-stage1.yml | 16 +++---- .../templates-official/jobs/source-build.yml | 8 ++++ .../post-build/post-build.yml | 8 ++-- .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ .../templates/job/publish-build-assets.yml | 21 +++++---- eng/common/templates/job/source-build.yml | 8 ++++ .../templates/job/source-index-stage1.yml | 11 ++--- eng/common/templates/jobs/source-build.yml | 8 ++++ .../templates/post-build/post-build.yml | 8 ++-- .../post-build/setup-maestro-vars.yml | 28 ++++++------ .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../templates/steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ global.json | 2 +- 20 files changed, 301 insertions(+), 63 deletions(-) create mode 100644 eng/common/templates-official/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates-official/steps/get-delegation-sas.yml create mode 100644 eng/common/templates-official/steps/get-federated-access-token.yml create mode 100644 eng/common/templates/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates/steps/get-delegation-sas.yml create mode 100644 eng/common/templates/steps/get-federated-access-token.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0ef64be84b..a948de8e7b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - e6f70c7dd528f05cd28cec2a179d58c22e91d9ac + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 5a3a32ea8d..238945cb5a 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -2,7 +2,6 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, [Parameter(Mandatory=$true)][string] $AzdoToken, - [Parameter(Mandatory=$true)][string] $MaestroToken, [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, @@ -31,13 +30,13 @@ try { } & $darc add-build-to-channel ` - --id $buildId ` - --publishing-infra-version $PublishingInfraVersion ` - --default-channels ` - --source-branch main ` - --azdev-pat $AzdoToken ` - --bar-uri $MaestroApiEndPoint ` - --password $MaestroToken ` + --id $buildId ` + --publishing-infra-version $PublishingInfraVersion ` + --default-channels ` + --source-branch main ` + --azdev-pat "$AzdoToken" ` + --bar-uri "$MaestroApiEndPoint" ` + --ci ` @optionalParams if ($LastExitCode -ne 0) { diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 589ac80a18..ba3e7df815 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -76,13 +76,16 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -137,14 +140,16 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f193dfbe23..f983033bb0 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -62,6 +68,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - template: /eng/common/templates-official/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 43ee0c202f..60dfb6b2d1 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -23,7 +23,7 @@ jobs: value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates-official/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} @@ -34,7 +34,8 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 + image: windows.vs2022.amd64 + os: windows steps: - ${{ each preStep in parameters.preSteps }}: @@ -70,16 +71,13 @@ jobs: scriptType: 'ps' scriptLocation: 'inlineScript' inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - script: | - echo "Client ID: $(ARM_CLIENT_ID)" - echo "ID Token: $(ARM_ID_TOKEN)" - echo "Tenant ID: $(ARM_TENANT_ID)" az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) displayName: "Login to Azure" - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index \ No newline at end of file + displayName: Upload stage1 artifacts to source index diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index 08e5db9bb1..5cf6a269c0 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates-official/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index da1f40958b..0dfa387e7b 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -272,14 +272,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 0000000000..93a8394a66 --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 0000000000..c0e8f91317 --- /dev/null +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 0000000000..e3786cef6d --- /dev/null +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 8ec0151def..57a41f0a3e 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -74,13 +74,16 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -133,14 +136,16 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 8a3deef2b7..c0ff472b69 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -61,6 +67,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - template: /eng/common/templates/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 43ee0c202f..0b6bb89dc7 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -70,16 +70,13 @@ jobs: scriptType: 'ps' scriptLocation: 'inlineScript' inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - script: | - echo "Client ID: $(ARM_CLIENT_ID)" - echo "ID Token: $(ARM_ID_TOKEN)" - echo "Tenant ID: $(ARM_TENANT_ID)" az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) displayName: "Login to Azure" - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index \ No newline at end of file + displayName: Upload stage1 artifacts to source index diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index a15b07eb51..5f46bfa895 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index aba44a25a3..2db4933468 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -268,14 +268,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0c87f149a4..64b9abc685 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -11,13 +11,14 @@ steps: artifactName: ReleaseConfigs checkDownloadedFiles: true - - task: PowerShell@2 + - task: AzureCLI@2 name: setReleaseVars displayName: Set Release Configs Vars inputs: - targetType: inline - pwsh: true - script: | + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | try { if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt @@ -31,15 +32,16 @@ steps: $AzureDevOpsBuildId = $Env:Build_BuildId } else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + . $(Build.SourcesDirectory)\eng\common\tools.ps1 + $darc = Get-Darc + $buildInfo = & $darc get-build ` + --id ${{ parameters.BARBuildId }} ` + --extended ` + --output-format json ` + --ci ` + | convertFrom-Json - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId + $BarId = ${{ parameters.BARBuildId }} $Channels = $Env:PromoteToMaestroChannels -split "," $Channels = $Channels -join "][" $Channels = "[$Channels]" @@ -65,6 +67,4 @@ steps: exit 1 } env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 0000000000..54dc9416c5 --- /dev/null +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 0000000000..c0e8f91317 --- /dev/null +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 0000000000..c8c49cc0e8 --- /dev/null +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file diff --git a/global.json b/global.json index 63925fe917..6d86fe094b 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24266.3" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24360.5" } } From e3059b2fc5aad4cf8de79f0d5d78dab2fbd6074c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 09:06:34 -0500 Subject: [PATCH 24/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240717.1 (#341) Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.24257.2 -> To Version 8.0.0-alpha.1.24367.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a948de8e7b..7c6f79cd39 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/source-build-reference-packages - 6ed73280a6d70f7e7ac39c86f2abe8c10983f0bb + 68d6cef51f1c82d71b435af0f040d72fdd1a782f From fb970eccb0a9cae3092464e29cbabda0d4115049 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:37:08 +0000 Subject: [PATCH 25/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240722.3 (#348) [release/8.0] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7c6f79cd39..aa9a485e41 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/source-build-reference-packages - 68d6cef51f1c82d71b435af0f040d72fdd1a782f + 30ed464acd37779c64e9dc652d4460543ebf9966 From fb5e370fa90df1671638651566b4b674ee486e75 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:57:28 +0000 Subject: [PATCH 26/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240815.1 (#384) [release/8.0] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index aa9a485e41..e930469225 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/source-build-reference-packages - 30ed464acd37779c64e9dc652d4460543ebf9966 + fe3794a68bd668d36d4d5014a9e6c9d22c0e6d86 From d4feb7e49067fc9bbf7dfb9fa76a326c33fa0595 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:59:30 +0000 Subject: [PATCH 27/33] Update dependencies from https://github.com/dotnet/arcade build 20241008.1 (#385) [release/8.0] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/sdl/NuGet.config | 4 ++-- eng/common/sdl/execute-all-sdl-tools.ps1 | 4 +--- eng/common/sdl/init-sdl.ps1 | 8 -------- eng/common/sdl/sdl.ps1 | 4 +++- .../job/publish-build-assets.yml | 2 +- .../templates-official/post-build/post-build.yml | 2 +- .../templates-official/steps/execute-sdl.yml | 2 -- .../steps/get-federated-access-token.yml | 14 +++++++++++++- eng/common/templates/job/publish-build-assets.yml | 2 +- eng/common/templates/post-build/post-build.yml | 2 +- eng/common/templates/steps/execute-sdl.yml | 7 ++++--- .../templates/steps/get-federated-access-token.yml | 14 +++++++++++++- eng/common/templates/steps/telemetry-start.yml | 2 +- eng/common/tools.ps1 | 2 +- global.json | 4 ++-- 16 files changed, 46 insertions(+), 31 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e930469225..aefd4f9f6b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + e5b13e054339e41d422212a0ecaf24fec20cb5a1 diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config index 3849bdb3cf..5bfbb02ef0 100644 --- a/eng/common/sdl/NuGet.config +++ b/eng/common/sdl/NuGet.config @@ -5,11 +5,11 @@ - + - + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 4715d75e97..81ded5b7f4 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -6,7 +6,6 @@ Param( [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located - [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list # format. @@ -75,7 +74,7 @@ try { } Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -GuardianLoggerLevel $GuardianLoggerLevel } $gdnFolder = Join-Path $workingDirectory '.gdn' @@ -104,7 +103,6 @@ try { -TargetDirectory $targetDirectory ` -GdnFolder $gdnFolder ` -ToolsList $tools ` - -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` -GuardianLoggerLevel $GuardianLoggerLevel ` -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 3ac1d92b37..588ff8e22f 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -3,7 +3,6 @@ Param( [string] $Repository, [string] $BranchName='master', [string] $WorkingDirectory, - [string] $AzureDevOpsAccessToken, [string] $GuardianLoggerLevel='Standard' ) @@ -21,14 +20,7 @@ $ci = $true # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' -# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file -$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) -$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" -$zipFile = "$WorkingDirectory/gdn.zip" - Add-Type -AssemblyName System.IO.Compression.FileSystem -$gdnFolder = (Join-Path $WorkingDirectory '.gdn') try { # if the folder does not exist, we'll do a guardian init and push it to the remote repository diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 index 648c5068d7..7fe603fe99 100644 --- a/eng/common/sdl/sdl.ps1 +++ b/eng/common/sdl/sdl.ps1 @@ -4,6 +4,8 @@ function Install-Gdn { [Parameter(Mandatory=$true)] [string]$Path, + [string]$Source = "https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json", + # If omitted, install the latest version of Guardian, otherwise install that specific version. [string]$Version ) @@ -19,7 +21,7 @@ function Install-Gdn { $ci = $true . $PSScriptRoot\..\tools.ps1 - $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") + $argumentList = @("install", "Microsoft.Guardian.Cli.win-x64", "-Source $Source", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") if ($Version) { $argumentList += "-Version $Version" diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index ba3e7df815..0117328800 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -149,7 +149,7 @@ jobs: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index 0dfa387e7b..b81b8770b3 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -281,7 +281,7 @@ stages: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml index 07426fde05..301d5c591e 100644 --- a/eng/common/templates-official/steps/execute-sdl.yml +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -9,8 +9,6 @@ parameters: steps: - task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml index e3786cef6d..55e33bd38f 100644 --- a/eng/common/templates-official/steps/get-federated-access-token.yml +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -3,6 +3,12 @@ parameters: type: string - name: outputVariableName type: string +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' # Resource to get a token for. Common values include: # - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps # - 'https://storage.azure.com/' for storage @@ -10,10 +16,16 @@ parameters: - name: resource type: string default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false steps: - task: AzureCLI@2 displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} inputs: azureSubscription: ${{ parameters.federatedServiceConnection }} scriptType: 'pscore' @@ -25,4 +37,4 @@ steps: exit 1 } Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 57a41f0a3e..cc2b346ba8 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -145,7 +145,7 @@ jobs: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 2db4933468..c3b6a3012f 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -277,7 +277,7 @@ stages: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 07426fde05..fe0ebf8c90 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -9,8 +9,6 @@ parameters: steps: - task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' @@ -36,16 +34,19 @@ steps: displayName: Execute SDL (Overridden) continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + env: + GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} -GuardianCliLocation $(GuardianCliLocation) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + env: + GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: # We want to publish the Guardian results and configuration for easy diagnosis. However, the diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml index c8c49cc0e8..55e33bd38f 100644 --- a/eng/common/templates/steps/get-federated-access-token.yml +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -3,6 +3,12 @@ parameters: type: string - name: outputVariableName type: string +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' # Resource to get a token for. Common values include: # - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps # - 'https://storage.azure.com/' for storage @@ -10,10 +16,16 @@ parameters: - name: resource type: string default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false steps: - task: AzureCLI@2 displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} inputs: azureSubscription: ${{ parameters.federatedServiceConnection }} scriptType: 'pscore' @@ -25,4 +37,4 @@ steps: exit 1 } Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml index 32c01ef0b5..6abbcb33a6 100644 --- a/eng/common/templates/steps/telemetry-start.yml +++ b/eng/common/templates/steps/telemetry-start.yml @@ -8,7 +8,7 @@ parameters: steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: - - task: AzureKeyVault@1 + - task: AzureKeyVault@2 inputs: azureSubscription: 'HelixProd_KeyVault' KeyVaultName: HelixProdKV diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index eb188cfda4..a2dedaa529 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -892,7 +892,7 @@ function IsWindowsPlatform() { } function Get-Darc($version) { - $darcPath = "$TempDir\darc\$(New-Guid)" + $darcPath = "$TempDir\darc\$([guid]::NewGuid())" if ($version -ne $null) { & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host } else { diff --git a/global.json b/global.json index 6d86fe094b..d4283f8ff4 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "8.0.101" + "dotnet": "8.0.110" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24360.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24508.1" } } From 3b85d089311e89b47758ba6a84eeb79374476dc8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 22:07:32 +0000 Subject: [PATCH 28/33] Update dependencies from https://github.com/dotnet/arcade build 20241016.1 (#396) [release/8.0] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- .../templates-official/steps/get-delegation-sas.yml | 11 ++++++++++- eng/common/templates/steps/get-delegation-sas.yml | 11 ++++++++++- global.json | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index aefd4f9f6b..15c5e3267f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - e5b13e054339e41d422212a0ecaf24fec20cb5a1 + f7fb1fec01b91be69e4dcc5290a0bff3f28e214f diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml index c0e8f91317..c690cc0a07 100644 --- a/eng/common/templates-official/steps/get-delegation-sas.yml +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -28,7 +28,16 @@ steps: # Calculate the expiration of the SAS token and convert to UTC $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") - $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads + # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 + $sas = "" + do { + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + } while($sas.IndexOf('/') -ne -1) if ($LASTEXITCODE -ne 0) { Write-Error "Failed to generate SAS token." diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml index c0e8f91317..c690cc0a07 100644 --- a/eng/common/templates/steps/get-delegation-sas.yml +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -28,7 +28,16 @@ steps: # Calculate the expiration of the SAS token and convert to UTC $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") - $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads + # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 + $sas = "" + do { + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + } while($sas.IndexOf('/') -ne -1) if ($LASTEXITCODE -ne 0) { Write-Error "Failed to generate SAS token." diff --git a/global.json b/global.json index d4283f8ff4..cfcbcd6904 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.110" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24508.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24516.1" } } From 4db185f73e5c970b2547a320f10b8820c6be2c8a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 9 Jan 2025 22:02:16 +0000 Subject: [PATCH 29/33] Update dependencies from https://github.com/dotnet/arcade build 20241223.2 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24516.1 -> To Version 8.0.0-beta.24623.2 --- eng/Version.Details.xml | 4 +-- eng/common/cross/toolchain.cmake | 61 ++++++++++++++++---------------- eng/common/tools.ps1 | 2 +- eng/common/tools.sh | 2 +- global.json | 2 +- 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 15c5e3267f..e08bf6a2bc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - f7fb1fec01b91be69e4dcc5290a0bff3f28e214f + 278b1dda181ab18a9fbed73da998e50d128eae21 diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 0998e875e5..dafabdcaef 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -40,7 +40,7 @@ if(TARGET_ARCH_NAME STREQUAL "arm") set(TOOLCHAIN "arm-linux-gnueabihf") endif() if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") + set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR aarch64) @@ -49,7 +49,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") elseif(LINUX) set(TOOLCHAIN "aarch64-linux-gnu") if(TIZEN) - set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu") endif() elseif(FREEBSD) set(triple "aarch64-unknown-freebsd12") @@ -58,7 +58,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) set(TOOLCHAIN "arm-linux-gnueabi") if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") + set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi") endif() elseif(TARGET_ARCH_NAME STREQUAL "armv6") set(CMAKE_SYSTEM_PROCESSOR armv6l) @@ -95,7 +95,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") elseif(LINUX) set(TOOLCHAIN "x86_64-linux-gnu") if(TIZEN) - set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu") endif() elseif(FREEBSD) set(triple "x86_64-unknown-freebsd12") @@ -112,7 +112,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") set(TOOLCHAIN "i686-linux-gnu") endif() if(TIZEN) - set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu") endif() else() message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!") @@ -124,26 +124,25 @@ endif() # Specify include paths if(TIZEN) - if(TARGET_ARCH_NAME STREQUAL "arm") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf) - endif() - if(TARGET_ARCH_NAME STREQUAL "armel") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) - endif() - if(TARGET_ARCH_NAME STREQUAL "arm64") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) - endif() - if(TARGET_ARCH_NAME STREQUAL "x86") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu) - endif() - if(TARGET_ARCH_NAME STREQUAL "x64") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu) + function(find_toolchain_dir prefix) + # Dynamically find the version subdirectory + file(GLOB DIRECTORIES "${prefix}/*") + list(GET DIRECTORIES 0 FIRST_MATCH) + get_filename_component(TOOLCHAIN_VERSION ${FIRST_MATCH} NAME) + + set(TIZEN_TOOLCHAIN_PATH "${prefix}/${TOOLCHAIN_VERSION}" PARENT_SCOPE) + endfunction() + + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") + find_toolchain_dir("${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + else() + find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() + + message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}") + + include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++) + include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN}) endif() if(ANDROID) @@ -265,21 +264,21 @@ endif() if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") if(TIZEN) - add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") endif() elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$") if(TIZEN) - add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") - add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-Wl,--rpath-link=${TIZEN_TOOLCHAIN_PATH}") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) @@ -288,10 +287,10 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") endif() add_toolchain_linker_flag(-m32) if(TIZEN) - add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") endif() elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index a2dedaa529..60352ede19 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -321,7 +321,7 @@ function InstallDotNet([string] $dotnetRoot, $variations += @($installParameters) $dotnetBuilds = $installParameters.Clone() - $dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public" + $dotnetbuilds.AzureFeed = "https://ci.dot.net/public" $variations += @($dotnetBuilds) if ($runtimeSourceFeed) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 3392e3a999..b9b329ce37 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -234,7 +234,7 @@ function InstallDotNet { local public_location=("${installParameters[@]}") variations+=(public_location) - local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://dotnetbuilds.azureedge.net/public") + local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://ci.dot.net/public") variations+=(dotnetbuilds) if [[ -n "${6:-}" ]]; then diff --git a/global.json b/global.json index cfcbcd6904..7127093ab7 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.110" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24516.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24623.2" } } From c301b8babe9ea6aeac4452ccb4dde45cae769282 Mon Sep 17 00:00:00 2001 From: Michael Simons Date: Fri, 10 Jan 2025 08:20:24 -0600 Subject: [PATCH 30/33] Update docker-creds-provider to address vulnerable system.text.json dependency. (#434) * Update docker-creds-provider to address CG alerts * Backport UpdateJson support --- .gitmodules | 9 +-- Directory.Build.props | 4 ++ build.sh | 7 ++ eng/Build.props | 3 + eng/Version.Details.xml | 20 ++++++ eng/Versions.props | 4 ++ eng/tasks/Directory.Build.props | 26 +++++++ ...soft.DotNet.SourceBuild.Tasks.XPlat.csproj | 26 +++++++ .../UpdateJson.cs | 69 +++++++++++++++++++ repo-projects/Directory.Build.targets | 1 + .../docker-creds-provider-2.2.1.proj | 7 -- ...-2.2.0.proj => docker-creds-provider.proj} | 2 +- repo-projects/docker-creds-provider.targets | 1 + src/docker-creds-provider | 1 + src/docker-creds-provider-2.2.0 | 1 - src/docker-creds-provider-2.2.1 | 1 - 16 files changed, 166 insertions(+), 16 deletions(-) create mode 100644 eng/tasks/Directory.Build.props create mode 100644 eng/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj create mode 100644 eng/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs delete mode 100644 repo-projects/docker-creds-provider-2.2.1.proj rename repo-projects/{docker-creds-provider-2.2.0.proj => docker-creds-provider.proj} (66%) create mode 160000 src/docker-creds-provider delete mode 160000 src/docker-creds-provider-2.2.0 delete mode 160000 src/docker-creds-provider-2.2.1 diff --git a/.gitmodules b/.gitmodules index 210cff8930..9d056e139d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,15 +16,12 @@ [submodule "src/azure-activedirectory-identitymodel-extensions-for-dotnet"] path = src/azure-activedirectory-identitymodel-extensions-for-dotnet url = https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet.git -[submodule "src/docker-creds-provider-2.2.0"] - path = src/docker-creds-provider-2.2.0 - url = https://github.com/mthalman/docker-creds-provider -[submodule "src/docker-creds-provider-2.2.1"] - path = src/docker-creds-provider-2.2.1 - url = https://github.com/mthalman/docker-creds-provider [submodule "xunit"] path = src/xunit url = https://github.com/xunit/xunit [submodule "abstractions-xunit"] path = src/abstractions-xunit url = https://github.com/xunit/abstractions.xunit +[submodule "src/docker-creds-provider"] + path = src/docker-creds-provider + url = https://github.com/mthalman/docker-creds-provider diff --git a/Directory.Build.props b/Directory.Build.props index 5309541c68..34c9d33b0d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -30,6 +30,10 @@ $(ProjectDir)patches/ $(NETCoreSdkVersion) + $(MSBuildThisFileDirectory)eng/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ + $(XPlatTasksDir)bin/$(Configuration)/ + $(XPlatTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll + $(DotNetCliToolDir)sdk/$(SDK_VERSION)/ + + https://github.com/dotnet/msbuild + b5265ef370a651f8c3458110b804e5cbf869eeb5 + + + + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime + 9cb3b725e3ad2b57ddc9fb2dd48d2d170563a8f5 + + + + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime + 9cb3b725e3ad2b57ddc9fb2dd48d2d170563a8f5 + + https://github.com/dotnet/arcade f7fb1fec01b91be69e4dcc5290a0bff3f28e214f diff --git a/eng/Versions.props b/eng/Versions.props index 766b606e59..5f2d44a4f1 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,4 +8,8 @@ alpha 1 + + + 17.8.5 + diff --git a/eng/tasks/Directory.Build.props b/eng/tasks/Directory.Build.props new file mode 100644 index 0000000000..98448e986d --- /dev/null +++ b/eng/tasks/Directory.Build.props @@ -0,0 +1,26 @@ + + + + + + AnyCPU + + true + + + + + + + + + + diff --git a/eng/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj b/eng/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj new file mode 100644 index 0000000000..17a1271005 --- /dev/null +++ b/eng/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj @@ -0,0 +1,26 @@ + + + + net8.0 + $(XPlatTasksBinDir) + + + + + False + False + + + + + + + + + + + + + + diff --git a/eng/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs b/eng/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs new file mode 100644 index 0000000000..9820e4af23 --- /dev/null +++ b/eng/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + // Takes a path to a path to a json file and a + // string that represents a dotted path to an attribute + // and updates that attribute with the new value provided. + public class UpdateJson : Task + { + [Required] + public string JsonFilePath { get; set; } + + [Required] + public string PathToAttribute { get; set; } + + [Required] + public string NewAttributeValue { get; set; } + + public bool SkipUpdateIfMissingKey { get; set; } + + public override bool Execute() + { + JObject jsonObj = JObject.Parse(File.ReadAllText(JsonFilePath)); + + string[] escapedPathToAttributeParts = PathToAttribute.Replace("\\.", "\x1F").Split('.'); + for (int i = 0; i < escapedPathToAttributeParts.Length; ++i) + { + escapedPathToAttributeParts[i] = escapedPathToAttributeParts[i].Replace("\x1F", "."); + } + UpdateAttribute(jsonObj, escapedPathToAttributeParts, NewAttributeValue); + + File.WriteAllText(JsonFilePath, jsonObj.ToString()); + return true; + } + + private void UpdateAttribute(JToken jsonObj, string[] path, string newValue) + { + string pathItem = path[0]; + if (jsonObj[pathItem] == null) + { + string message = $"Path item [{nameof(PathToAttribute)}] not found in json file."; + if (SkipUpdateIfMissingKey) + { + Log.LogMessage(MessageImportance.Low, $"Skipping update: {message} {pathItem}"); + return; + } + throw new ArgumentException(message, pathItem); + } + + if (path.Length == 1) + { + jsonObj[pathItem] = newValue; + return; + } + + UpdateAttribute(jsonObj[pathItem], path.Skip(1).ToArray(), newValue); + } + } +} diff --git a/repo-projects/Directory.Build.targets b/repo-projects/Directory.Build.targets index aff7be9f8b..36a880c697 100644 --- a/repo-projects/Directory.Build.targets +++ b/repo-projects/Directory.Build.targets @@ -8,6 +8,7 @@ + diff --git a/repo-projects/docker-creds-provider-2.2.1.proj b/repo-projects/docker-creds-provider-2.2.1.proj deleted file mode 100644 index 5e8c00c009..0000000000 --- a/repo-projects/docker-creds-provider-2.2.1.proj +++ /dev/null @@ -1,7 +0,0 @@ - - - 2.2.1 - - - - diff --git a/repo-projects/docker-creds-provider-2.2.0.proj b/repo-projects/docker-creds-provider.proj similarity index 66% rename from repo-projects/docker-creds-provider-2.2.0.proj rename to repo-projects/docker-creds-provider.proj index 75f1ca5415..79d56d6644 100644 --- a/repo-projects/docker-creds-provider-2.2.0.proj +++ b/repo-projects/docker-creds-provider.proj @@ -1,6 +1,6 @@ - 2.2.0 + 2.2.4 diff --git a/repo-projects/docker-creds-provider.targets b/repo-projects/docker-creds-provider.targets index 585991996f..b8719a355c 100644 --- a/repo-projects/docker-creds-provider.targets +++ b/repo-projects/docker-creds-provider.targets @@ -6,6 +6,7 @@ + $(ProjectDirectory)/global.json $(ProjectDirectory)/src/Valleysoft.DockerCredsProvider/bin/$(Configuration)/ diff --git a/src/docker-creds-provider b/src/docker-creds-provider new file mode 160000 index 0000000000..6e1ecd0a80 --- /dev/null +++ b/src/docker-creds-provider @@ -0,0 +1 @@ +Subproject commit 6e1ecd0a80755f9f0e88dc23b98b52f51a77c65e diff --git a/src/docker-creds-provider-2.2.0 b/src/docker-creds-provider-2.2.0 deleted file mode 160000 index 5701f6667c..0000000000 --- a/src/docker-creds-provider-2.2.0 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5701f6667c1fbd805684857baaa860383bbdfed7 diff --git a/src/docker-creds-provider-2.2.1 b/src/docker-creds-provider-2.2.1 deleted file mode 160000 index b381eafbee..0000000000 --- a/src/docker-creds-provider-2.2.1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b381eafbeecb1039f5839fc98ef45e7b3e52dee9 From e9f8fe6ca3f330ec6491e420c32f5265c3d6757f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 13:49:55 -0600 Subject: [PATCH 31/33] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20250110.2 (#439) Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.24415.1 -> To Version 8.0.0-alpha.1.25060.2 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 5 +++++ eng/Version.Details.xml | 4 ++-- eng/Versions.props | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/NuGet.config b/NuGet.config index 5691790f8b..fbc43e1448 100644 --- a/NuGet.config +++ b/NuGet.config @@ -2,6 +2,11 @@ + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 044f0641cd..bbab6eca70 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -26,9 +26,9 @@ 278b1dda181ab18a9fbed73da998e50d128eae21 - + https://github.com/dotnet/source-build-reference-packages - fe3794a68bd668d36d4d5014a9e6c9d22c0e6d86 + f9542c50beaefc38dd9d7ec9ea38d54fd154f21a diff --git a/eng/Versions.props b/eng/Versions.props index 5f2d44a4f1..07dc2320b9 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,7 +8,6 @@ alpha 1 - 17.8.5 From dc30cd1ec22f198d658e011c14525d4d65873991 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 13:50:23 -0600 Subject: [PATCH 32/33] Update dependencies from https://github.com/dotnet/arcade build 20250110.1 (#440) Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24623.2 -> To Version 8.0.0-beta.25060.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- eng/common/templates-official/job/publish-build-assets.yml | 2 +- eng/common/templates-official/post-build/common-variables.yml | 2 +- eng/common/templates/job/job.yml | 2 +- global.json | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index bbab6eca70..ba033ef192 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -21,9 +21,9 @@ 9cb3b725e3ad2b57ddc9fb2dd48d2d170563a8f5 - + https://github.com/dotnet/arcade - 278b1dda181ab18a9fbed73da998e50d128eae21 + c255aae7f2b128fa20a4441f0e192c3c53561621 diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 0117328800..b2ccd9df68 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -86,7 +86,7 @@ jobs: arguments: > -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index c24193acfc..173914f236 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -7,7 +7,7 @@ variables: # Default Maestro++ API Endpoint and API Version - name: MaestroApiEndPoint - value: "https://maestro-prod.westus2.cloudapp.azure.com" + value: "https://maestro.dot.net" - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 8ec5c4f2d9..e295031c09 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -128,7 +128,7 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@3 + - task: MicroBuildSigningPlugin@4 displayName: Install MicroBuild plugin inputs: signType: $(_SignType) diff --git a/global.json b/global.json index 7127093ab7..02fb60a4df 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.110" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24623.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25060.1" } } From 16bcad1c13be082bd52ce178896d1119a73081a9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:31:52 +0000 Subject: [PATCH 33/33] Add source-build team to CODEOWNERS (#466) Co-authored-by: Matt Thalman --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 1319ee24dd..8f4e63ba60 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Users referenced in this file will automatically be requested as reviewers for PRs that modify the given paths. # See https://help.github.com/articles/about-code-owners/ -* @MichaelSimons @mthalman +* @dotnet/source-build