From b15934d5a9e6bf97b047f63239cc21ba1c15cdd4 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 20 Jan 2025 14:26:04 +0100 Subject: [PATCH] ADLS: Support Vended Credentials (#1520) First version with `Fsspec`. Will check with PyArrow tomorrow, but that one seems to be lacking the SAS token: https://arrow.apache.org/docs/cpp/api/filesystem.html#azure-filesystem Closes #1146 --- pyiceberg/io/fsspec.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 62e9b92342..5ac5ce7d4c 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -179,6 +179,16 @@ def _gs(properties: Properties) -> AbstractFileSystem: def _adls(properties: Properties) -> AbstractFileSystem: from adlfs import AzureBlobFileSystem + for key, sas_token in { + key.replace(f"{ADLS_SAS_TOKEN}.", ""): value + for key, value in properties.items() + if key.startswith(ADLS_SAS_TOKEN) and key.endswith(".windows.net") + }.items(): + if ADLS_ACCOUNT_NAME not in properties: + properties[ADLS_ACCOUNT_NAME] = key.split(".")[0] + if ADLS_SAS_TOKEN not in properties: + properties[ADLS_SAS_TOKEN] = sas_token + return AzureBlobFileSystem( connection_string=properties.get(ADLS_CONNECTION_STRING), account_name=properties.get(ADLS_ACCOUNT_NAME),