From f7305973906792e61b1ad51c39917a29f5b58789 Mon Sep 17 00:00:00 2001 From: Mattias Karlsson Date: Sat, 9 Nov 2024 23:05:30 +0100 Subject: [PATCH] (GH-4391) GHA cross-job download artifact support * fixes #4391 --- .github/workflows/build.yml | 13 +++- .../Build/GitHubActionsCommandsFixture.cs | 63 ++++++++++++++----- .../Artifact/GitHubActionsArtifactService.cs | 31 ++++++++- .../Commands/Artifact/ListArtifactsRequest.cs | 20 ++++++ .../Artifact/ListArtifactsResponse.cs | 27 ++++++++ .../GitHubActions/GitHubActionsProvider.cake | 18 +++++- 6 files changed, 153 insertions(+), 19 deletions(-) create mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Artifact/ListArtifactsRequest.cs create mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Artifact/ListArtifactsResponse.cs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8089bfce3e..a6b811e2cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,8 +7,19 @@ on: - develop - hotfix/* jobs: + prepare: + name: Prepare integration tests + runs-on: ubuntu-latest + steps: + - run: echo "Cake Integration Tests" > cake-integration-tests.txt + - uses: actions/upload-artifact@v4 + with: + name: cake-integration-tests + path: cake-integration-tests.txt + build: name: Build + needs: prepare runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -34,7 +45,7 @@ jobs: - name: Run Cake script id: build-cake - uses: cake-build/cake-action@v1 + uses: cake-build/cake-action@master with: target: Run-Integration-Tests cake-version: tool-manifest diff --git a/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsCommandsFixture.cs b/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsCommandsFixture.cs index 58d1d92da4..23e083bcc2 100644 --- a/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsCommandsFixture.cs +++ b/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsCommandsFixture.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using Cake.Common.Build.GitHubActions.Commands; using Cake.Common.Build.GitHubActions.Commands.Artifact; -using Cake.Common.Build.GitHubActions.Data; using Cake.Common.Tests.Fakes; using Cake.Core; using Cake.Core.IO; @@ -22,21 +21,45 @@ internal sealed class GitHubActionsCommandsFixture : HttpMessageHandler private const string ArtifactUrl = GitHubActionsInfoFixture.ActionResultsUrl + "twirp/github.actions.results.api.v1.ArtifactService/"; private const string CreateArtifactUrl = ArtifactUrl + "CreateArtifact"; private const string FinalizeArtifactUrl = ArtifactUrl + "FinalizeArtifact"; - private const string GetSignedArtifactURLurl = ArtifactUrl + "GetSignedArtifactURL"; + private const string GetSignedArtifactURLUrl = ArtifactUrl + "GetSignedArtifactURL"; + private const string ListArtifactsUrl = ArtifactUrl + "ListArtifacts"; private const string UploadFileUrl = "https://cake.build.net/actions-results/a9d82106-d5d5-4310-8f60-0bfac035cf02/workflow-job-run-1d849a45-2f30-5fbb-3226-b730a17a93af/artifacts/91e64594182918fa8012cdbf7d1a4f801fa0c35f485c3277268aad8e3f45377c.zip?sig=upload"; private const string DownloadFileUrl = "https://cake.build.net/actions-results/a9d82106-d5d5-4310-8f60-0bfac035cf02/workflow-job-run-1d849a45-2f30-5fbb-3226-b730a17a93af/artifacts/91e64594182918fa8012cdbf7d1a4f801fa0c35f485c3277268aad8e3f45377c.zip?sig=download"; - private const string CreateArtifactResponse = @"{ - ""ok"": true, - ""signed_upload_url"": """ + UploadFileUrl + @""" -}"; - private const string FinalizeArtifactResponse = @"{ - ""ok"": true, - ""artifact_id"": ""1991105334"" -}"; - private const string GetSignedArtifactURLResponse = @"{ - ""name"": ""artifact"", - ""signed_url"": """ + DownloadFileUrl + @""" -}"; + private const string CreateArtifactResponse = + $$""" + { + "ok": true, + "signed_upload_url": "{{UploadFileUrl}}" + } + """; + private const string FinalizeArtifactResponse = + """ + { + "ok": true, + "artifact_id": "1991105334" + } + """; + private const string GetSignedArtifactURLResponse = + $$""" + { + "name": "artifact", + "signed_url": "{{DownloadFileUrl}}" + } + """; + private const string ListArtifactsResponse = + $$""" + { + "artifacts": [ + { + "workflow_run_backend_id": "b9e28153-ca20-4b86-91dd-09e8f644efdf", + "workflow_job_run_backend_id": "1d849a45-2f30-5fbb-3226-b730a17a93af", + "database_id": "1", + "name": "artifact", + "created_at": "2024-11-09T21:53:00.7110204+00:00" + } + ] + } + """; private GitHubActionsInfoFixture GitHubActionsInfoFixture { get; } private ICakeEnvironment Environment { get; } @@ -122,7 +145,7 @@ protected override async Task SendAsync(HttpRequestMessage // Get Signed Artifact Url case { - RequestUri: { AbsoluteUri: GetSignedArtifactURLurl }, + RequestUri: { AbsoluteUri: GetSignedArtifactURLUrl }, Method: { Method: "POST" }, }: { @@ -189,6 +212,16 @@ protected override async Task SendAsync(HttpRequestMessage return Ok(); } + // List Artifacts + case + { + RequestUri: { AbsoluteUri: ListArtifactsUrl }, + Method: { Method: "POST" } + }: + { + return Ok(new StringContent(ListArtifactsResponse)); + } + // Download File case { diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GitHubActionsArtifactService.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GitHubActionsArtifactService.cs index dc686e6ace..ee0f5f6796 100644 --- a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GitHubActionsArtifactService.cs +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GitHubActionsArtifactService.cs @@ -30,16 +30,43 @@ internal record GitHubActionsArtifactService( private static readonly Uri CreateArtifactUrl = new Uri("CreateArtifact", UriKind.Relative); private static readonly Uri FinalizeArtifactUrl = new Uri("FinalizeArtifact", UriKind.Relative); private static readonly Uri GetSignedArtifactURLUrl = new Uri("GetSignedArtifactURL", UriKind.Relative); + private static readonly Uri ListArtifactsUrl = new Uri("ListArtifacts", UriKind.Relative); internal async Task DownloadArtifactFiles( string artifactName, DirectoryPath directoryPath) + { + var listArtifactsResponse = await ListArtifacts( + artifactName); + + if (listArtifactsResponse.Artifacts.FirstOrDefault(artifact => artifact.Name == artifactName) + is { WorkflowRunBackendId.Length: > 0 } and { WorkflowJobRunBackendId.Length: > 0 } artifact) + { + var signedArtifactURLResponse = await GetSignedArtifactURL(artifact.WorkflowRunBackendId, artifact.WorkflowJobRunBackendId, artifactName); + + await DownloadArtifact(signedArtifactURLResponse.SignedUrl, directoryPath); + } + else + { + throw new CakeException($"Artifact {artifactName} not found."); + } + } + + private async Task ListArtifacts( + string nameFilter = null, + long? idFilter = null) { GetWorkflowBackendIds(out var workflowRunBackendId, out var workflowJobRunBackendId); - var (_, signedUrl) = await GetSignedArtifactURL(workflowRunBackendId, workflowJobRunBackendId, artifactName); + var listArtifactsRequest = new ListArtifactsRequest( + workflowRunBackendId, + workflowJobRunBackendId, + nameFilter, + idFilter); - await DownloadArtifact(signedUrl, directoryPath); + return await PostArtifactService( + ListArtifactsUrl, + listArtifactsRequest); } private async Task DownloadArtifact(string signedUrl, DirectoryPath directoryPath) diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/ListArtifactsRequest.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/ListArtifactsRequest.cs new file mode 100644 index 0000000000..fe06c1fa52 --- /dev/null +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/ListArtifactsRequest.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace Cake.Common.Build.GitHubActions.Commands.Artifact +{ +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + internal record ListArtifactsRequest( + [property: JsonPropertyName("workflow_run_backend_id")] + string WorkflowRunBackendId, + [property: JsonPropertyName("workflow_job_run_backend_id")] + string WorkflowJobRunBackendId, + [property: JsonPropertyName("name_filter")] + string NameFilter = null, + [property: JsonPropertyName("id_filter")] + long? IdFilter = null); +#pragma warning restore SA1313 // Parameter names should begin with lower-case letter +} diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/ListArtifactsResponse.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/ListArtifactsResponse.cs new file mode 100644 index 0000000000..28559b6f51 --- /dev/null +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/ListArtifactsResponse.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Text.Json.Serialization; + +namespace Cake.Common.Build.GitHubActions.Commands.Artifact +{ +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + internal record ListArtifactsResponse( + [property: JsonPropertyName("artifacts")] + ListArtifactsResponse.MonolithArtifact[] Artifacts) + { + internal record MonolithArtifact( + [property: JsonPropertyName("workflow_run_backend_id")] + string WorkflowRunBackendId, + [property: JsonPropertyName("workflow_job_run_backend_id")] + string WorkflowJobRunBackendId, + [property: JsonPropertyName("database_id")] + string DatabaseId, + [property: JsonPropertyName("name")] + string Name, + [property: JsonPropertyName("created_at")] + DateTimeOffset CreatedAt); + } +} \ No newline at end of file diff --git a/tests/integration/Cake.Common/Build/GitHubActions/GitHubActionsProvider.cake b/tests/integration/Cake.Common/Build/GitHubActions/GitHubActionsProvider.cake index 421ddc7f08..f6a5b9ab90 100644 --- a/tests/integration/Cake.Common/Build/GitHubActions/GitHubActionsProvider.cake +++ b/tests/integration/Cake.Common/Build/GitHubActions/GitHubActionsProvider.cake @@ -137,6 +137,21 @@ Task("Cake.Common.Build.GitHubActionsProvider.Commands.DownloadArtifact") Assert.True(FileHashEquals(data.AssemblyPath, targetArtifactPath), $"{data.AssemblyPath.FullPath}=={targetArtifactPath.FullPath}"); }); +Task("Cake.Common.Build.GitHubActionsProvider.Commands.DownloadArtifact.PreviousJob") + .Does(async () => { + // Given + var targetPath = Paths.Temp.Combine("./Cake.Common.Build.GitHubActionsProvider.Commands.DownloadArtifact.PreviousJob"); + EnsureDirectoryExists(targetPath); + var targetArtifactPath = targetPath.CombineWithFilePath("cake-integration-tests.txt"); + + // When + await GitHubActions.Commands.DownloadArtifact("cake-integration-tests", targetPath); + + // Then + Assert.True(System.IO.File.Exists(targetArtifactPath.FullPath), $"{targetArtifactPath.FullPath} Missing"); + Assert.Equal("Cake Integration Tests\n", System.IO.File.ReadAllText(targetArtifactPath.FullPath)); +}); + Task("Cake.Common.Build.GitHubActionsProvider.Environment.Runner.Architecture") .Does(() => { // Given / When @@ -193,7 +208,8 @@ if (GitHubActions.Environment.Runtime.IsRuntimeAvailable) gitHubActionsProviderTask .IsDependentOn("Cake.Common.Build.GitHubActionsProvider.Commands.UploadArtifact.File") .IsDependentOn("Cake.Common.Build.GitHubActionsProvider.Commands.UploadArtifact.Directory") - .IsDependentOn("Cake.Common.Build.GitHubActionsProvider.Commands.DownloadArtifact"); + .IsDependentOn("Cake.Common.Build.GitHubActionsProvider.Commands.DownloadArtifact") + .IsDependentOn("Cake.Common.Build.GitHubActionsProvider.Commands.DownloadArtifact.PreviousJob"); } public class GitHubActionsData