From 08b9a0432423614f6ca4c5845e18fae26b433bbe Mon Sep 17 00:00:00 2001
From: Eric O'Connor <eric@oco.nnor.org>
Date: Mon, 18 Sep 2023 22:20:23 -0700
Subject: [PATCH] Enable streaming upload in S3 : put_object

There were two problems:

1) Request did not allow an IO
2) Signing did not permit an unsigned payload
    (https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html)
---
 src/utilities/request.jl |  8 +++++++-
 src/utilities/sign.jl    | 18 ++++++++----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index e083017e67..b5ed7a231e 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -66,7 +66,7 @@ Base.@kwdef mutable struct Request
     request_method::String
 
     headers::AbstractDict{String,String} = LittleDict{String,String}()
-    content::Union{String,Vector{UInt8}} = ""
+    content::Union{String,Vector{UInt8},IO} = ""
     resource::String = ""
     url::String = ""
 
@@ -121,6 +121,12 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
         "EC2ThrottledException",
     ]
 
+    if isa(request.content, IO)
+        request.headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"
+    else
+        request.headers["Content-MD5"] = base64encode(
+            digest(MD_MD5, request.content))
+    end
     request.headers["User-Agent"] = user_agent[]
     request.headers["Host"] = HTTP.URI(request.url).host
     stream = @something request.response_stream IOBuffer()
diff --git a/src/utilities/sign.jl b/src/utilities/sign.jl
index 026b416023..2fd73cabb3 100644
--- a/src/utilities/sign.jl
+++ b/src/utilities/sign.jl
@@ -63,20 +63,18 @@ function sign_aws4!(aws::AbstractAWSConfig, request::Request, time::DateTime)
     # Authentication scope string...
     authentication_scope = join(authentication_scope, "/")
 
-    # SHA256 hash of content...
-    content_hash = bytes2hex(digest(MD_SHA256, request.content))
+    # SHA256 hash of content if data is in memory
+    if !haskey(request.headers, "x-amz-content-sha256")
+        content_hash = bytes2hex(digest(MD_SHA256, request.content))
+        request.headers["x-amz-content-sha256"] = content_hash
+    else
+        content_hash = "UNSIGNED-PAYLOAD"
+    end
 
     # HTTP headers...
     delete!(request.headers, "Authorization")
 
-    merge!(
-        request.headers,
-        Dict(
-            "x-amz-content-sha256" => content_hash,
-            "x-amz-date" => datetime,
-            "Content-MD5" => base64encode(digest(MD_MD5, request.content)),
-        ),
-    )
+    merge!(request.headers, Dict("x-amz-date" => datetime))
 
     if !isempty(creds.token)
         request.headers["x-amz-security-token"] = creds.token