Skip to content
Open
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/early-hotels-peel.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@e2b/python-sdk': patch
'e2b': patch
---

fix: set Content-Length on presigned PUT uploads for S3 compatibility
20 changes: 16 additions & 4 deletions packages/js-sdk/src/template/buildApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,24 @@ export async function uploadFile(
resolveSymlinks
)

// The compiler assumes this is Web fetch API, but it's actually Node.js fetch API
// Buffer the gzipped tar to determine Content-Length.
// S3 presigned PUT URLs do not support Transfer-Encoding: chunked,
// and the compressed size is unknowable without actually compressing.
let body: Buffer
{
const chunks: Buffer[] = []
for await (const chunk of uploadStream as unknown as AsyncIterable<Buffer>) {
chunks.push(chunk)
}
body = Buffer.concat(chunks)
}

const res = await fetch(url, {
method: 'PUT',
// @ts-expect-error
body: uploadStream,
duplex: 'half',
body,
headers: {
'Content-Length': String(body.length),
},
})

if (!res.ok) {
Expand Down
3 changes: 2 additions & 1 deletion packages/js-sdk/src/template/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -385,10 +385,11 @@ export async function tarFileStream(
}

/**
* Create a tar stream for upload using chunked transfer encoding.
* Create a tar stream for upload.
*
* @param fileName Glob pattern for files to include
* @param fileContextPath Base directory for resolving file paths
* @param ignorePatterns Ignore patterns to exclude from the archive
* @param resolveSymlinks Whether to follow symbolic links
* @returns A readable stream of the gzipped tar archive
*/
Expand Down
14 changes: 11 additions & 3 deletions packages/python-sdk/e2b/template_async/build_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,20 @@ async def upload_file(
stack_trace: Optional[TracebackType],
):
try:
tar_buffer = tar_file_stream(
file_name, context_path, ignore_patterns, resolve_symlinks
loop = asyncio.get_event_loop()
tar_buffer = await loop.run_in_executor(
None, tar_file_stream, file_name, context_path, ignore_patterns, resolve_symlinks
)

# Use bytes with explicit Content-Length.
# S3 presigned PUT URLs do not support Transfer-Encoding: chunked.
body = tar_buffer.getvalue()
client = api_client.get_async_httpx_client()
response = await client.put(url, content=tar_buffer.getvalue())
response = await client.put(
url,
content=body,
headers={"Content-Length": str(len(body))},
)
response.raise_for_status()
except httpx.HTTPStatusError as e:
raise FileUploadException(f"Failed to upload file: {e}").with_traceback(
Expand Down
10 changes: 9 additions & 1 deletion packages/python-sdk/e2b/template_sync/build_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,16 @@ def upload_file(
tar_buffer = tar_file_stream(
file_name, context_path, ignore_patterns, resolve_symlinks
)

# Use bytes with explicit Content-Length.
# S3 presigned PUT URLs do not support Transfer-Encoding: chunked.
body = tar_buffer.getvalue()
client = api_client.get_httpx_client()
response = client.put(url, content=tar_buffer.getvalue())
response = client.put(
url,
content=body,
headers={"Content-Length": str(len(body))},
)
response.raise_for_status()
except httpx.HTTPStatusError as e:
raise FileUploadException(f"Failed to upload file: {e}").with_traceback(
Expand Down
Loading