From ca696a9dc30dff3a9b7d58471315fa949521bc68 Mon Sep 17 00:00:00 2001 From: cscanlin Date: Wed, 26 Apr 2023 00:51:00 -0700 Subject: [PATCH 1/6] add initial s3_uri support Signed-off-by: cscanlin --- in/command.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/in/command.go b/in/command.go index ba6d21c9..740ce76c 100644 --- a/in/command.go +++ b/in/command.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strconv" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" "github.com/concourse/s3-resource/versions" ) @@ -28,6 +28,10 @@ func (up *RequestURLProvider) s3URL(request Request, remotePath string) string { return up.s3Client.URL(request.Source.Bucket, remotePath, request.Source.Private, request.Version.VersionID) } +func GetS3URI(request Request, remotePath string) string { + return "s3://" + request.Source.Bucket + remotePath +} + type Command struct { s3client s3resource.S3Client urlProvider RequestURLProvider @@ -56,6 +60,7 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e var versionNumber string var versionID string var url string + var s3_uri string var isInitialVersion bool var skipDownload bool @@ -152,6 +157,10 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e if err = command.writeURLFile(destinationDir, url); err != nil { return Response{}, err } + s3_uri = GetS3URI(request, remotePath) + if err = command.writeS3URIFile(destinationDir, s3_uri); err != nil { + return Response{}, err + } } err = command.writeVersionFile(versionNumber, destinationDir) @@ -182,6 +191,10 @@ func (command *Command) writeURLFile(destDir string, url string) error { return ioutil.WriteFile(filepath.Join(destDir, "url"), []byte(url), 0644) } +func (command *Command) writeS3URIFile(destDir string, s3_uri string) error { + return ioutil.WriteFile(filepath.Join(destDir, "s3_uri"), []byte(s3_uri), 0644) +} + func (command *Command) writeVersionFile(versionNumber string, destDir string) error { return ioutil.WriteFile(filepath.Join(destDir, "version"), []byte(versionNumber), 0644) } From 24e16c7cc3bfcf66bcd2fe6c0b7e0e7f12bf18d2 Mon Sep 17 00:00:00 2001 From: cscanlin Date: Fri, 28 Apr 2023 22:07:05 -0700 Subject: [PATCH 2/6] add support for writing the destination file_path to a file Signed-off-by: cscanlin --- in/command.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/in/command.go b/in/command.go index 740ce76c..26b24fa7 100644 --- a/in/command.go +++ b/in/command.go @@ -127,8 +127,8 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e return Response{}, err } + destinationPath := filepath.Join(destinationDir, path.Base(remotePath)) if request.Params.Unpack { - destinationPath := filepath.Join(destinationDir, path.Base(remotePath)) mime := archiveMimetype(destinationPath) if mime == "" { return Response{}, fmt.Errorf("not an archive: %s", destinationPath) @@ -139,6 +139,10 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e return Response{}, err } } + if err = command.WriteFilePathFile(destinationDir, destinationPath); err != nil { + return Response{}, err + } + } if request.Params.DownloadTags { @@ -195,6 +199,10 @@ func (command *Command) writeS3URIFile(destDir string, s3_uri string) error { return ioutil.WriteFile(filepath.Join(destDir, "s3_uri"), []byte(s3_uri), 0644) } +func (command *Command) WriteFilePathFile(destDir string, destinationPath string) error { + return ioutil.WriteFile(filepath.Join(destDir, "file_path"), []byte(destinationPath), 0644) +} + func (command *Command) writeVersionFile(versionNumber string, destDir string) error { return ioutil.WriteFile(filepath.Join(destDir, "version"), []byte(versionNumber), 0644) } From ad2e083e8f559f4c6b3c8ef7ac46d4e384db335f Mon Sep 17 00:00:00 2001 From: cscanlin Date: Fri, 28 Apr 2023 22:07:46 -0700 Subject: [PATCH 3/6] fix backwards arguments on writeVersionFile() Signed-off-by: cscanlin --- in/command.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/in/command.go b/in/command.go index 26b24fa7..26efdf9e 100644 --- a/in/command.go +++ b/in/command.go @@ -167,7 +167,7 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e } } - err = command.writeVersionFile(versionNumber, destinationDir) + err = command.writeVersionFile(destinationDir, versionNumber) if err != nil { return Response{}, err } @@ -203,7 +203,7 @@ func (command *Command) WriteFilePathFile(destDir string, destinationPath string return ioutil.WriteFile(filepath.Join(destDir, "file_path"), []byte(destinationPath), 0644) } -func (command *Command) writeVersionFile(versionNumber string, destDir string) error { +func (command *Command) writeVersionFile(destDir string, versionNumber string) error { return ioutil.WriteFile(filepath.Join(destDir, "version"), []byte(versionNumber), 0644) } From 1e04ede08a446470c01de7697707d396a7b66340 Mon Sep 17 00:00:00 2001 From: cscanlin Date: Fri, 28 Apr 2023 22:12:07 -0700 Subject: [PATCH 4/6] add README notes Signed-off-by: cscanlin --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c8f3967..718d3550 100644 --- a/README.md +++ b/README.md @@ -103,11 +103,15 @@ object's filename is the resulting version. Places the following files in the destination: -* `(filename)`: The file fetched from the bucket. +* `(filename)`: The file fetched from the bucket (if `skip_download` is not `true`). * `url`: A file containing the URL of the object. If `private` is true, this URL will be signed. +* `s3_uri`: A file containing the S3 URI of the object (for use with `aws cp`, etc.) + +* `file_path`: A file containing the absolute path to the downloaded filename (if `skip_download` is not `true`). Makes for easy use with the `load_var` step + * `version`: The version identified in the file name. * `tags.json`: The object's tags represented as a JSON object. Only written if `download_tags` is set to true. From afb3d8e2c69f0244f3bfcd1e7fcf7df38bc3ed9d Mon Sep 17 00:00:00 2001 From: cscanlin Date: Tue, 2 May 2023 10:53:35 -0700 Subject: [PATCH 5/6] add missing slash to GetS3URI Signed-off-by: cscanlin --- in/command.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/in/command.go b/in/command.go index 26efdf9e..a106c9ad 100644 --- a/in/command.go +++ b/in/command.go @@ -29,7 +29,7 @@ func (up *RequestURLProvider) s3URL(request Request, remotePath string) string { } func GetS3URI(request Request, remotePath string) string { - return "s3://" + request.Source.Bucket + remotePath + return "s3://" + request.Source.Bucket + "/" + remotePath } type Command struct { From a9dfe1c5cd0bf04dffe9e31a9ed35fe37ca32207 Mon Sep 17 00:00:00 2001 From: cscanlin Date: Wed, 3 May 2023 18:08:17 -0700 Subject: [PATCH 6/6] revert changes for file_path Signed-off-by: cscanlin --- README.md | 2 -- in/command.go | 10 +--------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/README.md b/README.md index 718d3550..5bb5f510 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,6 @@ Places the following files in the destination: * `s3_uri`: A file containing the S3 URI of the object (for use with `aws cp`, etc.) -* `file_path`: A file containing the absolute path to the downloaded filename (if `skip_download` is not `true`). Makes for easy use with the `load_var` step - * `version`: The version identified in the file name. * `tags.json`: The object's tags represented as a JSON object. Only written if `download_tags` is set to true. diff --git a/in/command.go b/in/command.go index a106c9ad..13d6ca45 100644 --- a/in/command.go +++ b/in/command.go @@ -127,8 +127,8 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e return Response{}, err } - destinationPath := filepath.Join(destinationDir, path.Base(remotePath)) if request.Params.Unpack { + destinationPath := filepath.Join(destinationDir, path.Base(remotePath)) mime := archiveMimetype(destinationPath) if mime == "" { return Response{}, fmt.Errorf("not an archive: %s", destinationPath) @@ -139,10 +139,6 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e return Response{}, err } } - if err = command.WriteFilePathFile(destinationDir, destinationPath); err != nil { - return Response{}, err - } - } if request.Params.DownloadTags { @@ -199,10 +195,6 @@ func (command *Command) writeS3URIFile(destDir string, s3_uri string) error { return ioutil.WriteFile(filepath.Join(destDir, "s3_uri"), []byte(s3_uri), 0644) } -func (command *Command) WriteFilePathFile(destDir string, destinationPath string) error { - return ioutil.WriteFile(filepath.Join(destDir, "file_path"), []byte(destinationPath), 0644) -} - func (command *Command) writeVersionFile(destDir string, versionNumber string) error { return ioutil.WriteFile(filepath.Join(destDir, "version"), []byte(versionNumber), 0644) }