Implement multipart copy and copying a particular version

src/AWSS3.jl

@@ -455,7 +455,7 @@ end
 """
     s3_copy([::AbstractAWSConfig], bucket, path; acl::AbstractString="",
             to_bucket=bucket, to_path=path, metadata::AbstractDict=SSDict(),
-            parse_response::Bool=true, kwargs...)
+            parse_response::Bool=true, version=nothing, kwargs...)
 
 Copy the object at `path` in `bucket` to `to_path` in `to_bucket`.
 
@@ -464,6 +464,8 @@ Copy the object at `path` in `bucket` to `to_path` in `to_bucket`.
     See [here](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl).
 - `metadata::Dict=`; `x-amz-meta-` headers.
 - `parse_response::Bool=`; when `false`, return raw `AWS.Response`
+- `version=`; when not `nothing`, the specific `versionId` of the source object to copy,
+  otherwise the latest version is copied.
 - `kwargs`; additional kwargs passed through into `S3.copy_object`
 
 # API Calls
@@ -483,6 +485,7 @@ function s3_copy(
     to_bucket=bucket,
     to_path=path,
     metadata::AbstractDict=SSDict(),
+    version::AbstractS3Version=nothing,
     parse_response::Bool=true,
     kwargs...,
 )
@@ -495,10 +498,15 @@ function s3_copy(
         headers["x-amz-acl"] = acl
     end
 
+    source = "$bucket/$path"
+    if version !== nothing
+        source *= "&versionId=" * escapeuri(version)
+    end
+
     response = S3.copy_object(
         to_bucket,
         to_path,
-        "$bucket/$path",
+        source,
         Dict("headers" => headers);
         aws_config=aws,
         kwargs...,
@@ -1075,6 +1083,31 @@ function s3_upload_part(
     return get_robust_case(Dict(response.headers), "ETag")
 end
 
+function s3_upload_part_copy(
+    aws::AbstractAWSConfig,
+    source,
+    upload,
+    part_number,
+    byte_range;
+    args=Dict{String,Any}(),
+    kwargs...,
+)
+    args["x-amz-copy-source-range"] = string(first(byte_range), '-', last(byte_range))
+
+    response = S3.upload_part_copy(
+        upload["Bucket"],
+        upload["Key"],
+        part_number,
+        upload["UploadId"],
+        source,
+        args;
+        aws_config=aws,
+        kwargs...,
+    )
+
+    return get_robust_case(Dict(response.headers), "ETag")
+end
+
 function s3_complete_multipart_upload(
     aws::AbstractAWSConfig,
     upload,
@@ -1145,6 +1178,70 @@ function s3_multipart_upload(
     return s3_complete_multipart_upload(aws, upload, tags; parse_response, kwargs...)
 end
 
+"""
+    s3_multipart_copy(aws::AbstractAWSConfig, bucket, path; to_bucket=bucket, to_path=path,
+                      part_size_mb=50, version=nothing, parse_response::Bool=true,
+                      kwargs...)
+
+Copy the object at `path` in `bucket` to `to_path` in `to_bucket` using a
+[multipart copy](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html).
+
+# Optional Arguments
+- `part_size_mb`: maximum size per uploaded part, in mebibytes (MiB).
+- `file_size_mb`: size in mebibytes of the object to copy. If `nothing`, the size is
+  determined using `s3_get_meta`.
+- `version`: when not `nothing`, the specific `versionId` of the source object to copy,
+  otherwise the latest version is copied.
+- `parse_response`: when `false`, return raw `AWS.Response`
+- `kwargs`: additional kwargs passed through to `s3_upload_part_copy` and `s3_complete_multipart_upload`
+
+# API Calls
+
+- [`HeadObject`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html) (if `file_size_mb` is not provided)
+- [`CreateMultipartUpload`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html)
+- [`UploadPartCopy`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html)
+- [`CompleteMultipartUpload`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html)
+
+# Permissions
+
+- [`s3:PutObject`](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazons3.html#amazons3-PutObject)
+- [`s3:GetObject`](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazons3.html#amazons3-GetObject)
+"""
+function s3_multipart_copy(
+    aws::AbstractAWSConfig,
+    bucket,
+    path;
+    to_bucket=bucket,
+    to_path=path,
+    part_size_mb::Integer=50,
+    file_size_mb::Union{Integer,Nothing}=nothing,
+    version::AbstractS3Version=nothing,
+    parse_response::Bool=true,
+    kwargs...,
+)
+    if file_size_mb === nothing
+        file_meta = s3_get_meta(aws, bucket, path; version)
+        file_size = parse(Int, get_robust_case(file_meta, "Content-Length"))
+    else
+        file_size = file_size_mb * 1024 * 1024
+    end
+
+    part_size = part_size_mb * 1024 * 1024
+
+    source = bucket * '/' * path
+    if version !== nothing
+        source *= "&versionId=" * version
+    end
+
+    upload = s3_begin_multipart_upload(aws, bucket, path)
+    tags = map(enumerate(0:part_size:file_size)) do (part, byte_offset)
+        byte_range = byte_offset:min(byte_offset + part_size - 1, file_size)
+        return s3_upload_part_copy(aws, source, upload, part, byte_range; kwargs...)
+    end
+
+    return s3_complete_multipart_upload(aws, upload, tags; parse_response, kwargs...)
+end
+
 using MbedTLS
 
 function _s3_sign_url_v2(

src/s3path.jl

@@ -722,6 +722,60 @@ function Base.write(
     end
 end
 
+const MiB = 1024 * 1024
+const GiB = MiB * 1024
+
+"""
+    Base.cp(src::S3Path, dest::S3Path; multipart::Bool=true, part_size_mb=50)
+
+Copy the object at `src` to `dest`.
+
+A multipart copy is used when `src` is 5 GiB or larger. A multipart copy can also be
+requested for objects larger than `part_size_mb` MiB by setting `multipart=true`.
+When `multipart=true` and `src` is larger than
+`part_size_mb` MiB, or when `src` is larger than 5 GiB, copying is performed using
+[`s3_multipart_copy`](@ref). Otherwise, [`s3_copy`](@ref) is used.
+"""
+function Base.cp(src::S3Path, dest::S3Path; multipart::Bool=true, part_size_mb=50)
+    if src.config !== nothing && dest.config !== nothing && src.config != dest.config
+        # Avoid breaking the case where the source and destination use different credentials
+        # by directly invoking the method that used to be called before this more specific
+        # method was added
+        return invoke(cp, Tuple{AbstractPath,AbstractPath}, src, dest)
+    end
+    config = @something(src.config, dest.config, global_aws_config())
+    head = s3_get_meta(config, src.bucket, src.key; src.version)
+    size = parse(Int, get_robust_case(head, "Content-Length"))
+    if (multipart && size > part_size * MiB) || size >= 5 * GiB
+        response = s3_multipart_copy(
+            config,
+            src.bucket,
+            src.key;
+            to_bucket=dest.bucket,
+            to_path=dest.key,
+            version=src.version,
+            part_size_mb,
+            file_size_mb=div(size, MiB),
+        )
+    else
+        response = s3_copy(
+            config,
+            src.bucket,
+            src.key;
+            to_bucket=dest.bucket,
+            to_path=dest.key,
+            version=src.version,
+        )
+    end
+    return S3Path(
+        dest.bucket,
+        dest.key;
+        dest.isdirectory,
+        dest.config,
+        version=HTTP.header(response.headers, "x-amz-version-id", nothing),
+    )
+end
+
 function FilePathsBase.mktmpdir(parent::S3Path)
     fp = parent / string(uuid4(), "/")
     return mkdir(fp)

test/awss3.jl

@@ -209,6 +209,26 @@ function awss3_tests(base_config)
         @test isa(result, AWS.Response)
     end
 
+    @testset "Multi-Part Copy" begin
+        config = assume_testset_role("MultipartUploadTestset"; base_config)
+        MIN_S3_CHUNK_SIZE = 5 * 1024 * 1024 # 5 MiB
+        src_key_name = "multi-part-key"
+        dest_key_name = "multi-part-key-copy"
+        result = s3_multipart_copy(
+            config,
+            bucket_name,
+            src_key_name;
+            to_bucket=bucket_name,
+            to_path=dest_key_name,
+            part_size_mb=MIN_S3_CHUNK_SIZE,
+        )
+        @test s3_exists(config, bucket_name, dest_key_name)
+        @test isa(result, LittleDict)
+        src_bytes = s3_get(config, bucket_name, src_key_name; raw=true)
+        dest_bytes = s3_get(config, bucket_name, dest_key_name; raw=true)
+        @test src_bytes == dest_bytes
+    end
+
     # these tests are needed because lack of functionality of the underlying AWS API makes certain
     # seemingly inane tasks incredibly tricky: for example checking if an "object" (file or
     # directory) exists is very subtle