add request timeout config

apache · Jan 23, 2025 · 1675f74 · 1675f74
1 parent 2cd4e78
commit 1675f74
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 0 deletions.
diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md
@@ -116,6 +116,7 @@ For the FileIO there are several configuration options available:
 | s3.region            | us-west-2                  | Configure the default region used to initialize an `S3FileSystem`. `PyArrowFileIO` attempts to automatically resolve the region for each S3 bucket, falling back to this value if resolution fails.                                                                                                           |
 | s3.proxy-uri         | <http://my.proxy.com:8080> | Configure the proxy server to be used by the FileIO.                                                                                                                                                                                                                    |
 | s3.connect-timeout   | 60.0                       | Configure socket connection timeout, in seconds.                                                                                                                                                                                                                        |
+| s3.request-timeout   | 60.0                       | Configure socket read timeouts on Windows and macOS, in seconds.                                                                                                                                                                                                        |
 | s3.force-virtual-addressing   | False                       | Whether to use virtual addressing of buckets. If true, then virtual addressing is always enabled. If false, then virtual addressing is only enabled if endpoint_override is empty. This can be used for non-AWS backends that only support virtual hosted-style access. |
 
 <!-- markdown-link-check-enable-->

diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py
@@ -61,6 +61,7 @@
 S3_REGION = "s3.region"
 S3_PROXY_URI = "s3.proxy-uri"
 S3_CONNECT_TIMEOUT = "s3.connect-timeout"
+S3_REQUEST_TIMEOUT = "s3.request-timeout"
 S3_SIGNER_URI = "s3.signer.uri"
 S3_SIGNER_ENDPOINT = "s3.signer.endpoint"
 S3_SIGNER_ENDPOINT_DEFAULT = "v1/aws/s3/sign"

diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py
@@ -62,6 +62,7 @@
     GCS_VERSION_AWARE,
     S3_ACCESS_KEY_ID,
     S3_CONNECT_TIMEOUT,
+    S3_REQUEST_TIMEOUT,
     S3_ENDPOINT,
     S3_PROXY_URI,
     S3_REGION,
@@ -150,6 +151,9 @@ def _s3(properties: Properties) -> AbstractFileSystem:
     if connect_timeout := properties.get(S3_CONNECT_TIMEOUT):
         config_kwargs["connect_timeout"] = float(connect_timeout)
 
+    if request_timeout := properties.get(S3_REQUEST_TIMEOUT):
+        config_kwargs["request_timeout"] = float(request_timeout)
+
     fs = S3FileSystem(client_kwargs=client_kwargs, config_kwargs=config_kwargs)
 
     for event_name, event_function in register_events.items():

diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py
@@ -100,6 +100,7 @@
     PYARROW_USE_LARGE_TYPES_ON_READ,
     S3_ACCESS_KEY_ID,
     S3_CONNECT_TIMEOUT,
+    S3_REQUEST_TIMEOUT,
     S3_ENDPOINT,
     S3_FORCE_VIRTUAL_ADDRESSING,
     S3_PROXY_URI,
@@ -394,6 +395,9 @@ def _initialize_oss_fs(self) -> FileSystem:
         if connect_timeout := self.properties.get(S3_CONNECT_TIMEOUT):
             client_kwargs["connect_timeout"] = float(connect_timeout)
 
+        if request_timeout := self.properties.get(S3_REQUEST_TIMEOUT):
+            client_kwargs["request_timeout"] = float(request_timeout)
+
         if role_arn := get_first_property_value(self.properties, S3_ROLE_ARN, AWS_ROLE_ARN):
             client_kwargs["role_arn"] = role_arn
 
@@ -438,6 +442,9 @@ def _initialize_s3_fs(self, netloc: Optional[str]) -> FileSystem:
         if connect_timeout := self.properties.get(S3_CONNECT_TIMEOUT):
             client_kwargs["connect_timeout"] = float(connect_timeout)
 
+        if request_timeout := self.properties.get(S3_REQUEST_TIMEOUT):
+            client_kwargs["request_timeout"] = float(request_timeout)
+
         if role_arn := get_first_property_value(self.properties, S3_ROLE_ARN, AWS_ROLE_ARN):
             client_kwargs["role_arn"] = role_arn