[master] Update dependencies from dotnet/arcade (#3025)

* Update dependencies from https://github.com/dotnet/arcade build 20190317.10

This change updates the following dependencies
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19167.10
- Microsoft.DotNet.SignTool - 1.0.0-beta.19167.10

* Updating LICENSE to match arcade's file.

* Adding copyright to the license file

* Update dependencies from https://github.com/dotnet/arcade build 20190320.2

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19170.2
- Microsoft.DotNet.SignTool - 1.0.0-beta.19170.2

* Update dependencies from https://github.com/dotnet/arcade build 20190321.3

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19171.3
- Microsoft.DotNet.SignTool - 1.0.0-beta.19171.3

* Update dependencies from https://github.com/dotnet/arcade build 20190321.6

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19171.6
- Microsoft.DotNet.SignTool - 1.0.0-beta.19171.6

LICENSE.TXT

@@ -2,6 +2,8 @@ The MIT License (MIT)
 
 Copyright (c) .NET Foundation and Contributors
 
+All rights reserved.
+
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
@@ -18,4 +20,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

eng/Version.Details.xml

@@ -3,13 +3,13 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19162.7">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19171.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>443dea11f8649fe12fedf60cfab0a4b2b20bd153</Sha>
+      <Sha>2eb45ff0eed6f86f1071b16b11a447cadeeeec59</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="1.0.0-beta.19162.7">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="1.0.0-beta.19171.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>443dea11f8649fe12fedf60cfab0a4b2b20bd153</Sha>
+      <Sha>2eb45ff0eed6f86f1071b16b11a447cadeeeec59</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -22,7 +22,7 @@
     <PlatformAbstractionsVersion>2.0.0</PlatformAbstractionsVersion>
     <SystemDiagnosticsFileVersionInfoVersion>4.0.0</SystemDiagnosticsFileVersionInfoVersion>
     <SystemReflectionMetadataVersion>1.5.0</SystemReflectionMetadataVersion>
-    <MicrosoftDotNetSignToolVersion>1.0.0-beta.19162.7</MicrosoftDotNetSignToolVersion>
+    <MicrosoftDotNetSignToolVersion>1.0.0-beta.19171.6</MicrosoftDotNetSignToolVersion>
   </PropertyGroup>
   <!-- Get .NET Framework reference assemblies from NuGet packages -->
   <PropertyGroup>

eng/common/PublishToPackageFeed.proj

@@ -11,11 +11,17 @@
   </PropertyGroup>
 
   <Import Project="$(MSBuildThisFileDirectory)DefaultVersions.props" Condition="Exists('$(MSBuildThisFileDirectory)DefaultVersions.props')" />
+
+  <!--
+    This won't be necessary once we solve this issue:
+    https://github.com/dotnet/arcade/issues/2266
+  -->
+  <Import Project="$(MSBuildThisFileDirectory)ArtifactsCategory.props" Condition="Exists('$(MSBuildThisFileDirectory)ArtifactsCategory.props')" />
 
   <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
 
   <Target Name="PublishToFeed">
-    <Error Condition="'$(TargetStaticFeed)' == ''" Text="TargetStaticFeed: Target feed for publishing assets wasn't provided." />
+    <Error Condition="'$(ArtifactsCategory)' == ''" Text="ArtifactsCategory: The artifacts' category produced by the build wasn't provided." />
     <Error Condition="'$(AccountKeyToStaticFeed)' == ''" Text="AccountKeyToStaticFeed: Account key for target feed wasn't provided." />
     <Error Condition="'$(ManifestsBasePath)' == ''" Text="Full path to asset manifests directory wasn't provided." />
     <Error Condition="'$(BlobBasePath)' == '' AND '$(PackageBasePath)' == ''" Text="A valid full path to BlobBasePath of PackageBasePath is required." />
@@ -26,7 +32,21 @@
     </ItemGroup>
 
     <Error Condition="'@(ManifestFiles)' == ''" Text="No manifest file was found in the provided path: $(ManifestsBasePath)" />
-
+
+    <!--
+      For now the type of packages being published will be informed for the whole build.
+      Eventually this will be specified on a per package basis:
+      TODO: https://github.com/dotnet/arcade/issues/2266
+    -->
+    <PropertyGroup>
+      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == '.NETCORE'">https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json</TargetStaticFeed>
+      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == '.NETCOREVALIDATION'">https://dotnetfeed.blob.core.windows.net/arcade-validation/index.json</TargetStaticFeed>
+    </PropertyGroup>
+
+    <Error 
+      Condition="'$(TargetStaticFeed)' == ''" 
+      Text="'$(ArtifactsCategory)' wasn't recognized as a valid artifact category. Valid categories are: '.NetCore' and '.NetCoreValidation'" />
+
     <!-- Iterate publishing assets from each manifest file. -->
     <PushArtifactsInManifestToFeed
       ExpectedFeedUrl="$(TargetStaticFeed)"

eng/common/generate-graph-files.ps1

@@ -3,14 +3,17 @@ Param(
   [Parameter(Mandatory=$true)][string] $gitHubPat,      # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed)
   [Parameter(Mandatory=$true)][string] $azdoPat,        # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed)
   [Parameter(Mandatory=$true)][string] $outputFolder,   # Where the graphviz.txt file will be created
-  [string] $darcVersion = '1.1.0-beta.19156.4',         # darc's version
+  [string] $darcVersion = '1.1.0-beta.19171.6',         # darc's version
+  [string] $graphvizVersion = '2.38',                   # GraphViz version
   [switch] $includeToolset                              # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about
                                                         # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
 )
 
 $ErrorActionPreference = "Stop"
 . $PSScriptRoot\tools.ps1
 
+Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1")
+
 function CheckExitCode ([string]$stage)
 {
   $exitCode = $LASTEXITCODE
@@ -27,27 +30,48 @@ try {
   . .\darc-init.ps1 -darcVersion $darcVersion
   CheckExitCode "Running darc-init"
 
+  $engCommonBaseDir = Join-Path $PSScriptRoot "native\"
+  $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory
+  $nativeToolBaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external"
+  $installBin = Join-Path $graphvizInstallDir "bin"
+
+  Write-Host "Installing dot..."
+  .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose
+
   $darcExe = "$env:USERPROFILE\.dotnet\tools"
   $darcExe = Resolve-Path "$darcExe\darc.exe"
 
   Create-Directory $outputFolder
 
+  # Generate 3 graph descriptions:
+  # 1. Flat with coherency information
+  # 2. Graphviz (dot) file
+  # 3. Standard dependency graph
   $graphVizFilePath = "$outputFolder\graphviz.txt"
-  $graphFilePath = "$outputFolder\graph.txt"
-  $options = "get-dependency-graph --graphviz '$graphVizFilePath' --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken --output-file $graphFilePath"
+  $graphVizImageFilePath = "$outputFolder\graph.png"
+  $normalGraphFilePath = "$outputFolder\graph-full.txt"
+  $flatGraphFilePath = "$outputFolder\graph-flat.txt"
+  $baseOptions = "get-dependency-graph --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken"
 
   if ($includeToolset) {
     Write-Host "Toolsets will be included in the graph..."
-    $options += " --include-toolset"
+    $baseOptions += " --include-toolset"
   }
 
-  Write-Host "Generating dependency graph..."
-  Invoke-Expression "& `"$darcExe`" $options"
-  CheckExitCode "Generating dependency graph"
+  Write-Host "Generating standard dependency graph..."
+  Invoke-Expression "& `"$darcExe`" $baseOptions --output-file $normalGraphFilePath"
+  CheckExitCode "Generating normal dependency graph"
+
+  Write-Host "Generating flat dependency graph and graphviz file..."
+  Invoke-Expression "& `"$darcExe`" $baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath"
+  CheckExitCode "Generating flat and graphviz dependency graph"
+
+  Write-Host "Generating graph image $graphVizFilePath"
+  $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe"
+  Invoke-Expression "& `"$dotFilePath`" -Tpng -o'$graphVizImageFilePath' `"$graphVizFilePath`""
+  CheckExitCode "Generating graphviz image"
 
-  $graph = Get-Content $graphVizFilePath
-  Set-Content $graphVizFilePath -Value "Paste the following digraph object in http://www.webgraphviz.com `r`n", $graph
-  Write-Host "'$graphVizFilePath' and '$graphFilePath' created!"
+  Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!"
 }
 catch {
   if (!$includeToolset) {
@@ -58,4 +82,6 @@ catch {
   Write-Host $_.Exception
   Write-Host $_.ScriptStackTrace
   ExitWithExitCode 1
+} finally {
+    Pop-Location
 }

eng/common/init-tools-native.ps1

@@ -98,10 +98,20 @@ try {
       Write-Verbose "Executing '$LocalInstallerCommand'"
       Invoke-Expression "$LocalInstallerCommand"
       if ($LASTEXITCODE -Ne "0") {
-        Write-Error "Execution failed"
-        exit 1
+        $errMsg = "$ToolName installation failed"
+        if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
+            Write-Warning $errMsg
+            $toolInstallationFailure = $true
+        } else {
+            Write-Error $errMsg
+            exit 1
+        }
       }
     }
+
+    if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
+        exit 1
+    }
   }
   else {
     Write-Host "No native tools defined in global.json"

eng/common/templates/job/publish-build-assets.yml

@@ -20,6 +20,9 @@ parameters:
   #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
   runAsPublic: false
 
+  # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
+  publishUsingPipelines: false
+
 jobs:
 - job: Asset_Registry_Publish
 
@@ -52,6 +55,7 @@ jobs:
           /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
           /p:BuildAssetRegistryToken=$(MaestroAccessToken)
           /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
+          /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
           /p:Configuration=$(_BuildConfig)
       condition: ${{ parameters.condition }}
       continueOnError: ${{ parameters.continueOnError }}

eng/common/templates/jobs/jobs.yml

@@ -13,6 +13,9 @@ parameters:
 
   # Optional: Enable publishing to the build asset registry
   enablePublishBuildAssets: false
+
+  # Optional: Enable publishing using release pipelines
+  enablePublishUsingPipelines: false
 
   graphFileGeneration:
     # Optional: Enable generating the graph files at the end of the build
@@ -73,6 +76,7 @@ jobs:
       pool:
         vmImage: vs2017-win2016
       runAsPublic: ${{ parameters.runAsPublic }}
+      publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
       enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
 
 - ${{ if and(eq(parameters.graphFileGeneration.enabled, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:

eng/common/templates/steps/send-to-helix.yml

@@ -1,8 +1,9 @@
+# Please remember to update the documentation if you make changes to these parameters!
 parameters:
   HelixSource: 'pr/default'              # required -- sources must start with pr/, official/, prodcon/, or agent/
   HelixType: 'tests/default/'            # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
   HelixBuild: $(Build.BuildNumber)       # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
-  HelixTargetQueues: ''                  # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
+  HelixTargetQueues: ''                  # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
   HelixAccessToken: ''                   # required -- access token to make Helix API requests; should be provided by the appropriate variable group
   HelixPreCommands: ''                   # optional -- commands to run before Helix work item execution
   HelixPostCommands: ''                  # optional -- commands to run after Helix work item execution
@@ -11,6 +12,7 @@ parameters:
   WorkItemTimeout: ''                    # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
   CorrelationPayloadDirectory: ''        # optional -- a directory to zip up and send to Helix as a correlation payload
   XUnitProjects: ''                      # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+  XUnitWorkItemTimeout: ''               # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects
   XUnitPublishTargetFramework: ''        # optional -- framework to use to publish your xUnit projects
   XUnitRuntimeTargetFramework: ''        # optional -- framework to use for the xUnit console runner
   XUnitRunnerVersion: ''                 # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
@@ -42,6 +44,7 @@ steps:
       WorkItemTimeout: ${{ parameters.WorkItemTimeout }}
       CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
       XUnitProjects: ${{ parameters.XUnitProjects }}
+      XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }}
       XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
       XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
       XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
@@ -70,6 +73,7 @@ steps:
       WorkItemTimeout: ${{ parameters.WorkItemTimeout }}
       CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
       XUnitProjects: ${{ parameters.XUnitProjects }}
+      XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }}
       XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
       XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
       XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
@@ -81,4 +85,4 @@ steps:
       Creator: ${{ parameters.Creator }}
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
-    continueOnError: ${{ parameters.continueOnError }}
+    continueOnError: ${{ parameters.continueOnError }}

global.json

@@ -6,6 +6,6 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19162.7"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19171.6"
   }
 }