Update dependencies from https://github.com/dotnet/arcade build 20240…

…531.1 (#2518)

[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,17 +3,17 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24278.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24281.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0c4e52c37a29b75f64646220e0cc237177ce23a2</Sha>
+      <Sha>e6b3f32f9855dccbe2447471c8f729b66f17d242</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.24278.1">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.24281.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0c4e52c37a29b75f64646220e0cc237177ce23a2</Sha>
+      <Sha>e6b3f32f9855dccbe2447471c8f729b66f17d242</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="9.0.0-beta.24278.1">
+    <Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="9.0.0-beta.24281.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0c4e52c37a29b75f64646220e0cc237177ce23a2</Sha>
+      <Sha>e6b3f32f9855dccbe2447471c8f729b66f17d242</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -15,6 +15,6 @@
     <SystemCollectionsImmutableVersion>8.0.0</SystemCollectionsImmutableVersion>
     <SystemIOHashingVersion>8.0.0</SystemIOHashingVersion>
     <MicrosoftBclTimeProviderVersion>8.0.0</MicrosoftBclTimeProviderVersion>
-    <MicrosoftDotNetXUnitExtensionsPackageVersion>9.0.0-beta.24278.1</MicrosoftDotNetXUnitExtensionsPackageVersion>
+    <MicrosoftDotNetXUnitExtensionsPackageVersion>9.0.0-beta.24281.1</MicrosoftDotNetXUnitExtensionsPackageVersion>
   </PropertyGroup>
 </Project>

eng/common/SetupNugetSources.ps1

@@ -1,31 +1,32 @@
-# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
-# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
+# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
+# disabled internal Maestro (darc-int*) feeds.
+# 
+# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. This credential
+# is added via the standard environment variable VSS_NUGET_EXTERNAL_FEED_ENDPOINTS. See
+# https://github.com/microsoft/artifacts-credprovider/tree/v1.1.1?tab=readme-ov-file#environment-variables for more details
 #
-# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
-# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential 
-# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
-#
-# This script needs to be called in every job that will restore packages and which the base repo has
-# private AzDO feeds in the NuGet.config.
-#
-# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
-# from the AzureDevOps-Artifact-Feeds-Pats variable group.
-#
-# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing
+# See example call for this script below.
 #
 #  - task: PowerShell@2
-#    displayName: Setup Private Feeds Credentials
+#    displayName: Setup Internal Feeds
 #    condition: eq(variables['Agent.OS'], 'Windows_NT')
 #    inputs:
 #      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
-#      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
-#    env:
-#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+#      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config
+#  - task: NuGetAuthenticate@1
+# 
+# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
+# This ensures that:
+# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
+# - The credential provider is installed
+#
+# This logic is also abstracted into enable-internal-sources.yml.
 
 [CmdletBinding()]
 param (
     [Parameter(Mandatory = $true)][string]$ConfigFile,
-    [Parameter(Mandatory = $true)][string]$Password
+    [string]$Password
 )
 
 $ErrorActionPreference = "Stop"
@@ -34,12 +35,23 @@ Set-StrictMode -Version 2.0
 
 . $PSScriptRoot\tools.ps1
 
+$feedEndpoints = $null
+
+# If a credential is provided, ensure that we don't overwrite the current set of
+# credentials that may have been provided by a previous call to the credential provider.
+if ($Password -and $null -ne $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS) {
+    $feedEndpoints = $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS | ConvertFrom-Json
+} elseif ($Password) {
+    $feedEndpoints = @{ endpointCredentials = @() }
+}
+
 # Add source entry to PackageSources
-function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $pwd) {
     $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
 
-    if ($packageSource -eq $null)
+    if ($null -eq $packageSource)
     {
+        Write-Host "`tAdding package source" $SourceName
         $packageSource = $doc.CreateElement("add")
         $packageSource.SetAttribute("key", $SourceName)
         $packageSource.SetAttribute("value", $SourceEndPoint)
@@ -48,58 +60,34 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern
     else {
         Write-Host "Package source $SourceName already present."
     }
-    AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
-}
-
-# Add a credential node for the specified source
-function AddCredential($creds, $source, $username, $pwd) {
-    # Looks for credential configuration for the given SourceName. Create it if none is found.
-    $sourceElement = $creds.SelectSingleNode($Source)
-    if ($sourceElement -eq $null)
-    {
-        $sourceElement = $doc.CreateElement($Source)
-        $creds.AppendChild($sourceElement) | Out-Null
-    }
 
-    # Add the <Username> node to the credential if none is found.
-    $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
-    if ($usernameElement -eq $null)
-    {
-        $usernameElement = $doc.CreateElement("add")
-        $usernameElement.SetAttribute("key", "Username")
-        $sourceElement.AppendChild($usernameElement) | Out-Null
+    if ($pwd) {
+        $feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $SourceEndPoint -pwd $pwd
     }
-    $usernameElement.SetAttribute("value", $Username)
+}
 
-    # Add the <ClearTextPassword> to the credential if none is found.
-    # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
-    #   -> https://github.com/NuGet/Home/issues/5526
-    $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
-    if ($passwordElement -eq $null)
-    {
-        $passwordElement = $doc.CreateElement("add")
-        $passwordElement.SetAttribute("key", "ClearTextPassword")
-        $sourceElement.AppendChild($passwordElement) | Out-Null
+# Add a new feed endpoint credential
+function AddCredential([array]$endpointCredentials, $source, $pwd) {
+    $endpointCredentials += @{
+        endpoint = $source;
+        password = $pwd
     }
-
-    $passwordElement.SetAttribute("value", $pwd)
+    return $endpointCredentials
 }
 
-function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) {
-    $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
+function InsertMaestroInternalFeedCredentials($Sources, $pwd) {
+    $maestroInternalSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
 
-    Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
-
-    ForEach ($PackageSource in $maestroPrivateSources) {
-        Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
-        AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd
+    ForEach ($PackageSource in $maestroInternalSources) {
+        Write-Host "`tAdding credential for Maestro's feed:" $PackageSource.Key
+        $feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $PackageSource.value -pwd $pwd
     }
 }
 
-function EnablePrivatePackageSources($DisabledPackageSources) {
-    $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
-    ForEach ($DisabledPackageSource in $maestroPrivateSources) {
-        Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
+function EnableInternalPackageSources($DisabledPackageSources) {
+    $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
+    ForEach ($DisabledPackageSource in $maestroInternalSources) {
+        Write-Host "`tEnsuring internal source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
         # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
         $DisabledPackageSources.RemoveChild($DisabledPackageSource)
     }
@@ -110,58 +98,53 @@ if (!(Test-Path $ConfigFile -PathType Leaf)) {
   ExitWithExitCode 1
 }
 
-if (!$Password) {
-    Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
-    ExitWithExitCode 1
-}
-
 # Load NuGet.config
 $doc = New-Object System.Xml.XmlDocument
 $filename = (Get-Item $ConfigFile).FullName
 $doc.Load($filename)
 
 # Get reference to <PackageSources> or create one if none exist already
 $sources = $doc.DocumentElement.SelectSingleNode("packageSources")
-if ($sources -eq $null) {
+if ($null -eq $sources) {
     $sources = $doc.CreateElement("packageSources")
     $doc.DocumentElement.AppendChild($sources) | Out-Null
 }
 
-# Looks for a <PackageSourceCredentials> node. Create it if none is found.
-$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
-if ($creds -eq $null) {
-    $creds = $doc.CreateElement("packageSourceCredentials")
-    $doc.DocumentElement.AppendChild($creds) | Out-Null
-}
-
 # Check for disabledPackageSources; we'll enable any darc-int ones we find there
 $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
-if ($disabledSources -ne $null) {
+if ($null -ne $disabledSources) {
     Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
-    EnablePrivatePackageSources -DisabledPackageSources $disabledSources
+    EnableInternalPackageSources -DisabledPackageSources $disabledSources
 }
 
-$userName = "dn-bot"
-
-# Insert credential nodes for Maestro's private feeds
-InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password
+if ($Password) {
+    InsertMaestroInternalFeedCredentials -Sources $sources -pwd $Password
+}
 
 # 3.1 uses a different feed url format so it's handled differently here
 $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
-if ($dotnet31Source -ne $null) {
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+if ($null -ne $dotnet31Source) {
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json" -pwd $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json" -pwd $Password
 }
 
 $dotnetVersions = @('5','6','7','8')
 
 foreach ($dotnetVersion in $dotnetVersions) {
     $feedPrefix = "dotnet" + $dotnetVersion;
     $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
-    if ($dotnetSource -ne $null) {
-        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
-        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+    if ($dotnetSource) {
+        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedprefix-internal/nuget/v3/index.json" -pwd $Password
+        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v3/index.json" -pwd $Password
     }
 }
 
-$doc.Save($filename)
+$doc.Save($filename)
+
+# If any credentials were added or altered, update the VSS_NUGET_EXTERNAL_FEED_ENDPOINTS environment variable
+if ($null -ne $feedEndpoints) {
+    # ci is set to true so vso logging commands will be used.
+    $ci = $true
+    Write-PipelineSetVariable -Name 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' -Value $($feedEndpoints | ConvertTo-Json) -IsMultiJobVariable $false
+    Write-PipelineSetVariable -Name 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' -Value "False" -IsMultiJobVariable $false
+}

eng/common/SetupNugetSources.sh

@@ -1,28 +1,27 @@
 #!/usr/bin/env bash
 
-# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
-# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+# This script adds internal feeds required to build commits that depend on intenral package sources. For instance,
+# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
+# disabled internal Maestro (darc-int*) feeds.
+# 
+# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
 #
-# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
-# under <packageSourceCredentials> for each Maestro's managed private feed. Two additional credential 
-# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
-#
-# This script needs to be called in every job that will restore packages and which the base repo has
-# private AzDO feeds in the NuGet.config.
-#
-# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
-# from the AzureDevOps-Artifact-Feeds-Pats variable group.
-#
-# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing.
+# See example call for this script below.
 #
 #  - task: Bash@3
-#    displayName: Setup Private Feeds Credentials
+#    displayName: Setup Internal Feeds
 #    inputs:
 #      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-#      arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+#      arguments: $(Build.SourcesDirectory)/NuGet.config
 #    condition: ne(variables['Agent.OS'], 'Windows_NT')
-#    env:
-#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+#  - task: NuGetAuthenticate@1
+#
+# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
+# This ensures that:
+# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
+# - The credential provider is installed.
+#
+# This logic is also abstracted into enable-internal-sources.yml.
 
 ConfigFile=$1
 CredToken=$2
@@ -48,11 +47,6 @@ if [ ! -f "$ConfigFile" ]; then
     ExitWithExitCode 1
 fi
 
-if [ -z "$CredToken" ]; then
-    Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT"
-    ExitWithExitCode 1
-fi
-
 if [[ `uname -s` == "Darwin" ]]; then
     NL=$'\\\n'
     TB=''
@@ -140,18 +134,20 @@ PackageSources+="$IFS"
 PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
 IFS=$PrevIFS
 
-for FeedName in ${PackageSources[@]} ; do
-    # Check if there is no existing credential for this FeedName
-    grep -i "<$FeedName>" $ConfigFile 
-    if [ "$?" != "0" ]; then
-        echo "Adding credentials for $FeedName."
+if [ "$CredToken" ]; then
+    for FeedName in ${PackageSources[@]} ; do
+        # Check if there is no existing credential for this FeedName
+        grep -i "<$FeedName>" $ConfigFile 
+        if [ "$?" != "0" ]; then
+            echo "Adding credentials for $FeedName."
 
-        PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
-        NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
+            PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
+            NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
 
-        sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
-    fi
-done
+            sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
+        fi
+    done
+fi
 
 # Re-enable any entries in disabledPackageSources where the feed name contains darc-int
 grep -i "<disabledPackageSources>" $ConfigFile

eng/common/core-templates/job/source-build.yml

@@ -33,6 +33,12 @@ parameters:
 
   is1ESPipeline: ''
 
+  # If set to true and running on a non-public project,
+  # Internal nuget and blob storage locations will be enabled.
+  # This is not enabled by default because many repositories do not need internal sources
+  # and do not need to have the required service connections approved in the pipeline.
+  enableInternalSources: false
+
 jobs:
 - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
   displayName: Source-Build (${{ parameters.platform.name }})
@@ -74,6 +80,13 @@ jobs:
   - ${{ if eq(parameters.is1ESPipeline, '') }}:
     - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
 
+  - ${{ if eq(parameters.enableInternalSources, true) }}:
+    - template: /eng/common/core-templates/steps/enable-internal-sources.yml
+      parameters:
+        is1ESPipeline: ${{ parameters.is1ESPipeline }}
+    - template: /eng/common/core-templates/steps/enable-internal-runtimes.yml
+      parameters:
+        is1ESPipeline: ${{ parameters.is1ESPipeline }}
   - template: /eng/common/core-templates/steps/source-build.yml
     parameters:
       is1ESPipeline: ${{ parameters.is1ESPipeline }}