From 1b781a01e58bd16f709a61cabf4041b12fed9613 Mon Sep 17 00:00:00 2001 From: James Myatt Date: Thu, 17 Jul 2025 15:10:21 +0100 Subject: [PATCH 1/2] Refactor do_connect for better logging And consistent ordering of credentials --- adlfs/spec.py | 123 +++++++++++++++++++++----------------------------- 1 file changed, 51 insertions(+), 72 deletions(-) diff --git a/adlfs/spec.py b/adlfs/spec.py index 13640a3c..40e45705 100644 --- a/adlfs/spec.py +++ b/adlfs/spec.py @@ -350,6 +350,13 @@ def __init__( max_concurrency = batch_size self.max_concurrency = max_concurrency + @property + def account_url(self) -> str: + if hasattr(self, "account_host"): + return f"https://{self.account_host}" + else: + return f"https://{self.account_name}.blob.core.windows.net" + @classmethod def _strip_protocol(cls, path: str): """ @@ -472,47 +479,7 @@ def do_connect(self): """ try: - if self.connection_string is not None: - self.service_client = AIOBlobServiceClient.from_connection_string( - conn_str=self.connection_string - ) - elif self.account_name is not None: - if hasattr(self, "account_host"): - self.account_url: str = f"https://{self.account_host}" - else: - self.account_url: str = ( - f"https://{self.account_name}.blob.core.windows.net" - ) - - creds = [self.credential, self.account_key] - if any(creds): - self.service_client = [ - AIOBlobServiceClient( - account_url=self.account_url, - credential=cred, - _location_mode=self.location_mode, - ) - for cred in creds - if cred is not None - ][0] - elif self.sas_token is not None: - if not self.sas_token.startswith("?"): - self.sas_token = f"?{self.sas_token}" - self.service_client = AIOBlobServiceClient( - account_url=self.account_url + self.sas_token, - credential=None, - _location_mode=self.location_mode, - ) - else: - # Fall back to anonymous login, and assume public container - self.service_client = AIOBlobServiceClient( - account_url=self.account_url - ) - else: - raise ValueError( - "Must provide either a connection_string or account_name with credentials!!" - ) - + self.service_client = self._get_service_client() except RuntimeError: loop = get_loop() asyncio.set_event_loop(loop) @@ -521,6 +488,46 @@ def do_connect(self): except Exception as e: raise ValueError(f"unable to connect to account for {e}") from e + def _get_service_client(self) -> AIOBlobServiceClient: + """Connect to the Asynchronous BlobServiceClient. + + Tries connection string first, then credentials and finally account key. + """ + # Shortcut for connection string + if self.connection_string is not None: + logger.info("Connect using connection string") + return AIOBlobServiceClient.from_connection_string( + conn_str=self.connection_string + ) + + if self.account_name is not None: + kwargs = { + "account_url": self.account_url, + "credential": None, + "_location_mode": self.location_mode, + } + creds = ["credential", "sync_credential", "account_key"] + for name in creds: + if (cred := getattr(self, name, None)) is not None: + logger.info("Connect using %s", name.replace("_", " ")) + kwargs["credential"] = cred + break + else: + if self.sas_token is not None: + logger.info("Connect using SAS token") + if not self.sas_token.startswith("?"): + self.sas_token = f"?{self.sas_token}" + kwargs["account_url"] = f'{kwargs["account_url"]}{self.sas_token}' + else: + logger.info("Connect using anonymous login") + # Fall back to anonymous login, and assume public container + + return AIOBlobServiceClient(**kwargs) + + raise ValueError( + "Must provide either a connection_string or account_name with credentials!!" + ) + def split_path( self, path, delimiter="/", return_container: bool = False, **kwargs ) -> Tuple[str, str, Optional[str]]: @@ -2030,43 +2037,15 @@ def close(self): def connect_client(self): """Connect to the Asynchronous BlobServiceClient, using user-specified connection details. - Tries credentials first, then connection string and finally account key Raises ------ ValueError if none of the connection details are available """ try: - if hasattr(self.fs, "account_host"): - self.fs.account_url: str = f"https://{self.fs.account_host}" - else: - self.fs.account_url: str = ( - f"https://{self.fs.account_name}.blob.core.windows.net" - ) - - creds = [self.fs.sync_credential, self.fs.account_key, self.fs.credential] - if any(creds): - self.container_client = [ - AIOBlobServiceClient( - account_url=self.fs.account_url, - credential=cred, - _location_mode=self.fs.location_mode, - ).get_container_client(self.container_name) - for cred in creds - if cred is not None - ][0] - elif self.fs.connection_string is not None: - self.container_client = AIOBlobServiceClient.from_connection_string( - conn_str=self.fs.connection_string - ).get_container_client(self.container_name) - elif self.fs.sas_token is not None: - self.container_client = AIOBlobServiceClient( - account_url=self.fs.account_url + self.fs.sas_token, credential=None - ).get_container_client(self.container_name) - else: - self.container_client = AIOBlobServiceClient( - account_url=self.fs.account_url - ).get_container_client(self.container_name) + self.container_client = self.fs._get_service_client().get_container_client( + self.container_name + ) except Exception as e: raise ValueError( From 7f594569be10ff2d9bb186b63481783142c8ca4e Mon Sep 17 00:00:00 2001 From: James Myatt Date: Thu, 17 Jul 2025 15:17:01 +0100 Subject: [PATCH 2/2] Fix docstrings --- adlfs/spec.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/adlfs/spec.py b/adlfs/spec.py index 40e45705..2fe4ef3e 100644 --- a/adlfs/spec.py +++ b/adlfs/spec.py @@ -471,7 +471,7 @@ def _get_default_azure_credential(self, **kwargs): def do_connect(self): """Connect to the BlobServiceClient, using user-specified connection details. - Tries credentials first, then connection string and finally account key + Tries connection string first, then credential and finally account key Raises ------ @@ -491,7 +491,7 @@ def do_connect(self): def _get_service_client(self) -> AIOBlobServiceClient: """Connect to the Asynchronous BlobServiceClient. - Tries connection string first, then credentials and finally account key. + Tries connection string first, then credential and finally account key """ # Shortcut for connection string if self.connection_string is not None: @@ -2037,6 +2037,7 @@ def close(self): def connect_client(self): """Connect to the Asynchronous BlobServiceClient, using user-specified connection details. + Tries connection string first, then credential and finally account key Raises ------