From f6a6c1d62811c52f86a42f8eec4f78ee9ca60d14 Mon Sep 17 00:00:00 2001 From: David F Little Date: Fri, 15 Jul 2022 15:44:32 +0000 Subject: [PATCH 1/5] robustly handle case variation in return value of s3_get_meta --- src/s3path.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/s3path.jl b/src/s3path.jl index 54b53344..afb0c808 100644 --- a/src/s3path.jl +++ b/src/s3path.jl @@ -333,6 +333,10 @@ function _walkpath!( end end +function get_robust_case(x, key) + haskey(x, key) && return x[key] + return x[lowercase(key)] +end """ stat(fp::S3Path) @@ -358,9 +362,9 @@ function Base.stat(fp::S3Path) # Example: "Thu, 03 Jan 2019 21:09:17 GMT" last_modified = DateTime( - resp["Last-Modified"][1:(end - 4)], dateformat"e, d u Y H:M:S" + get_robust_case(resp, "Last-Modified")[1:(end - 4)], dateformat"e, d u Y H:M:S" ) - s = parse(Int, resp["Content-Length"]) + s = parse(Int, get_robust_case(resp, "Content-Length")) blocks = ceil(Int, s / 4096) end From 0f5de1d14d021ca5ced27b6fbb130ed8bacf4e43 Mon Sep 17 00:00:00 2001 From: David F Little Date: Mon, 18 Jul 2022 14:37:04 +0000 Subject: [PATCH 2/5] apply `get_robust_path` to `s3_upload_part` --- src/AWSS3.jl | 8 +++++++- src/s3path.jl | 4 ---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/AWSS3.jl b/src/AWSS3.jl index f1579090..5914532d 100644 --- a/src/AWSS3.jl +++ b/src/AWSS3.jl @@ -58,6 +58,12 @@ const SSDict = Dict{String,String} const AbstractS3Version = Union{AbstractString,Nothing} const AbstractS3PathConfig = Union{AbstractAWSConfig,Nothing} +# Utility function to workaround https://github.com/JuliaCloud/AWS.jl/issues/547 +function get_robust_case(x, key) + haskey(x, key) && return x[key] + return x[lowercase(key)] +end + __init__() = FilePathsBase.register(S3Path) # Declare new `parse` function to avoid type piracy @@ -837,7 +843,7 @@ function s3_upload_part( kwargs..., ) - return Dict(response.headers)["ETag"] + return get_robust_case(Dict(response.headers), "ETag") end function s3_complete_multipart_upload( diff --git a/src/s3path.jl b/src/s3path.jl index afb0c808..7ce5bb8b 100644 --- a/src/s3path.jl +++ b/src/s3path.jl @@ -333,10 +333,6 @@ function _walkpath!( end end -function get_robust_case(x, key) - haskey(x, key) && return x[key] - return x[lowercase(key)] -end """ stat(fp::S3Path) From 8fb4973ae519af26136043bf5b82cb0c8ca5586c Mon Sep 17 00:00:00 2001 From: David Little Date: Mon, 18 Jul 2022 14:41:10 -0400 Subject: [PATCH 3/5] Update src/AWSS3.jl Co-authored-by: Eric Hanson <5846501+ericphanson@users.noreply.github.com> --- src/AWSS3.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/AWSS3.jl b/src/AWSS3.jl index 5914532d..c6129b2d 100644 --- a/src/AWSS3.jl +++ b/src/AWSS3.jl @@ -60,8 +60,9 @@ const AbstractS3PathConfig = Union{AbstractAWSConfig,Nothing} # Utility function to workaround https://github.com/JuliaCloud/AWS.jl/issues/547 function get_robust_case(x, key) - haskey(x, key) && return x[key] - return x[lowercase(key)] + lkey = lowercase(key) + haskey(x, lkey) && return x[lkey] + return x[key] end __init__() = FilePathsBase.register(S3Path) From 815e86b6b36a56007d5fbb74a784339e9e2dbb0c Mon Sep 17 00:00:00 2001 From: David F Little Date: Mon, 18 Jul 2022 18:48:50 +0000 Subject: [PATCH 4/5] bump version & add test --- Project.toml | 2 +- test/awss3.jl | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a6f1c183..99b17f4a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "AWSS3" uuid = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" -version = "0.9.6" +version = "0.9.7" [deps] AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" diff --git a/test/awss3.jl b/test/awss3.jl index a5c93b05..345695cd 100644 --- a/test/awss3.jl +++ b/test/awss3.jl @@ -3,6 +3,15 @@ function awss3_tests(config) "ocaws.jl.test." * lowercase(Dates.format(now(Dates.UTC), df)) end + @testset "Robust key selection" begin + lower_dict = Dict("foo-bar" => 1) + upper_dict = Dict("Foo-Bar" => 1) + @test AWSS3.get_robust_case(lower_dict, "Foo-Bar") == 1 + @test AWSS3.get_robust_case(upper_dict, "Foo-Bar") == 1 + @test AWSS3.get_robust_case(Dict(), "Foo-Bar") + @test_throws KeyError("Foo-Bar") AWSS3.get_robust_case(Dict(), "Foo-Bar") + end + @testset "Create Bucket" begin s3_create_bucket(config, bucket_name) @test bucket_name in s3_list_buckets(config) From 768bcbdfc763b41b0e46bd6d06061a42230a313c Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 18 Jul 2022 20:59:13 +0200 Subject: [PATCH 5/5] Update test/awss3.jl --- test/awss3.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/awss3.jl b/test/awss3.jl index 345695cd..526a212a 100644 --- a/test/awss3.jl +++ b/test/awss3.jl @@ -8,7 +8,6 @@ function awss3_tests(config) upper_dict = Dict("Foo-Bar" => 1) @test AWSS3.get_robust_case(lower_dict, "Foo-Bar") == 1 @test AWSS3.get_robust_case(upper_dict, "Foo-Bar") == 1 - @test AWSS3.get_robust_case(Dict(), "Foo-Bar") @test_throws KeyError("Foo-Bar") AWSS3.get_robust_case(Dict(), "Foo-Bar") end