Skip to content

Commit

Permalink
Enable streaming upload in S3 : put_object
Browse files Browse the repository at this point in the history
There were two problems:

1) Request did not allow an IO
2) Signing did not permit an unsigned payload
    (https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html)
  • Loading branch information
oconnore committed Sep 19, 2023
1 parent 100c30e commit 08b9a04
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 11 deletions.
8 changes: 7 additions & 1 deletion src/utilities/request.jl
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ Base.@kwdef mutable struct Request
request_method::String

headers::AbstractDict{String,String} = LittleDict{String,String}()
content::Union{String,Vector{UInt8}} = ""
content::Union{String,Vector{UInt8},IO} = ""
resource::String = ""
url::String = ""

Expand Down Expand Up @@ -121,6 +121,12 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
"EC2ThrottledException",
]

if isa(request.content, IO)
request.headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"
else
request.headers["Content-MD5"] = base64encode(
digest(MD_MD5, request.content))
end
request.headers["User-Agent"] = user_agent[]
request.headers["Host"] = HTTP.URI(request.url).host
stream = @something request.response_stream IOBuffer()
Expand Down
18 changes: 8 additions & 10 deletions src/utilities/sign.jl
Original file line number Diff line number Diff line change
Expand Up @@ -63,20 +63,18 @@ function sign_aws4!(aws::AbstractAWSConfig, request::Request, time::DateTime)
# Authentication scope string...
authentication_scope = join(authentication_scope, "/")

# SHA256 hash of content...
content_hash = bytes2hex(digest(MD_SHA256, request.content))
# SHA256 hash of content if data is in memory
if !haskey(request.headers, "x-amz-content-sha256")
content_hash = bytes2hex(digest(MD_SHA256, request.content))
request.headers["x-amz-content-sha256"] = content_hash
else
content_hash = "UNSIGNED-PAYLOAD"
end

# HTTP headers...
delete!(request.headers, "Authorization")

merge!(
request.headers,
Dict(
"x-amz-content-sha256" => content_hash,
"x-amz-date" => datetime,
"Content-MD5" => base64encode(digest(MD_MD5, request.content)),
),
)
merge!(request.headers, Dict("x-amz-date" => datetime))

if !isempty(creds.token)
request.headers["x-amz-security-token"] = creds.token
Expand Down

0 comments on commit 08b9a04

Please sign in to comment.