|
87 | 87 | ADLS_ACCOUNT_NAME,
|
88 | 88 | ADLS_BLOB_STORAGE_AUTHORITY,
|
89 | 89 | ADLS_BLOB_STORAGE_SCHEME,
|
| 90 | + ADLS_CLIENT_ID, |
| 91 | + ADLS_CLIENT_SECRET, |
90 | 92 | ADLS_DFS_STORAGE_AUTHORITY,
|
91 | 93 | ADLS_DFS_STORAGE_SCHEME,
|
92 | 94 | ADLS_SAS_TOKEN,
|
| 95 | + ADLS_TENANT_ID, |
93 | 96 | AWS_ACCESS_KEY_ID,
|
94 | 97 | AWS_REGION,
|
95 | 98 | AWS_ROLE_ARN,
|
@@ -501,6 +504,7 @@ def _initialize_s3_fs(self, netloc: Optional[str]) -> FileSystem:
|
501 | 504 | return S3FileSystem(**client_kwargs)
|
502 | 505 |
|
503 | 506 | def _initialize_azure_fs(self) -> FileSystem:
|
| 507 | + # https://arrow.apache.org/docs/python/generated/pyarrow.fs.AzureFileSystem.html |
504 | 508 | from packaging import version
|
505 | 509 |
|
506 | 510 | MIN_PYARROW_VERSION_SUPPORTING_AZURE_FS = "20.0.0"
|
@@ -535,6 +539,24 @@ def _initialize_azure_fs(self) -> FileSystem:
|
535 | 539 | if sas_token := self.properties.get(ADLS_SAS_TOKEN):
|
536 | 540 | client_kwargs["sas_token"] = sas_token
|
537 | 541 |
|
| 542 | + if client_id := self.properties.get(ADLS_CLIENT_ID): |
| 543 | + client_kwargs["client_id"] = client_id |
| 544 | + if client_secret := self.properties.get(ADLS_CLIENT_SECRET): |
| 545 | + client_kwargs["client_secret"] = client_secret |
| 546 | + if tenant_id := self.properties.get(ADLS_TENANT_ID): |
| 547 | + client_kwargs["tenant_id"] = tenant_id |
| 548 | + |
| 549 | + # Validate that all three are provided together for ClientSecretCredential |
| 550 | + credential_keys = ["client_id", "client_secret", "tenant_id"] |
| 551 | + provided_keys = [key for key in credential_keys if key in client_kwargs] |
| 552 | + if provided_keys and len(provided_keys) != len(credential_keys): |
| 553 | + missing_keys = [key for key in credential_keys if key not in client_kwargs] |
| 554 | + raise ValueError( |
| 555 | + f"client_id, client_secret, and tenant_id must all be provided together " |
| 556 | + f"to use ClientSecretCredential for Azure authentication. " |
| 557 | + f"Provided: {provided_keys}, Missing: {missing_keys}" |
| 558 | + ) |
| 559 | + |
538 | 560 | return AzureFileSystem(**client_kwargs)
|
539 | 561 |
|
540 | 562 | def _initialize_hdfs_fs(self, scheme: str, netloc: Optional[str]) -> FileSystem:
|
|
0 commit comments