Skip to content

Commit 02d019e

Browse files
CoderKevinZhangCoderKevinZhang
authored
Unexclude _artifacts folder (#33984)
* unexclude _artifacts folder * update * update
1 parent 47c67c0 commit 02d019e

File tree

9 files changed

+63
-52
lines changed

9 files changed

+63
-52
lines changed

‎sdk/ml/azure-ai-ml/azure/ai/ml/_artifacts/_artifact_utilities.py‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,9 @@
99
import uuid
1010
from datetime import datetime, timedelta
1111
from pathlib import Path
12-
from typing import Dict, Optional, Tuple, TypeVar, Union
12+
from typing import TYPE_CHECKING, Dict, Optional, Tuple, TypeVar, Union
13+
1314
from typing_extensions import Literal
14-
from azure.storage.blob import BlobSasPermissions, generate_blob_sas
15-
from azure.storage.filedatalake import FileSasPermissions, generate_file_sas
1615

1716
from azure.ai.ml._artifacts._blob_storage_helper import BlobStorageClient
1817
from azure.ai.ml._artifacts._gen2_storage_helper import Gen2StorageClient
@@ -47,6 +46,12 @@
4746
from azure.ai.ml.entities._datastore._constants import WORKSPACE_BLOB_STORE
4847
from azure.ai.ml.exceptions import ErrorTarget, ValidationException
4948
from azure.ai.ml.operations._datastore_operations import DatastoreOperations
49+
from azure.storage.blob import BlobSasPermissions, generate_blob_sas
50+
from azure.storage.filedatalake import FileSasPermissions, generate_file_sas
51+
52+
if TYPE_CHECKING:
53+
from azure.ai.ml.operations import DataOperations, EnvironmentOperations, FeatureSetOperations, ModelOperations
54+
from azure.ai.ml.operations._code_operations import CodeOperations
5055

5156
module_logger = logging.getLogger(__name__)
5257

@@ -60,7 +65,7 @@ def _get_datastore_name(*, datastore_name: Optional[str] = WORKSPACE_BLOB_STORE)
6065
datastore_name = remove_aml_prefix(datastore_name)
6166
if is_ARM_id_for_resource(datastore_name):
6267
datastore_name = get_resource_name_from_arm_id(datastore_name)
63-
return datastore_name
68+
return str(datastore_name)
6469

6570

6671
def get_datastore_info(
@@ -83,7 +88,7 @@ def get_datastore_info(
8388
:return: The dictionary with datastore info
8489
:rtype: Dict[Literal["storage_type", "storage_account", "account_url", "container_name"], str]
8590
"""
86-
datastore_info = {}
91+
datastore_info: Dict = {}
8792
if name:
8893
datastore = operations.get(name, include_secrets=credential is None)
8994
else:
@@ -538,12 +543,13 @@ def _check_and_upload_env_build_context(
538543
datastore_name=environment.datastore,
539544
show_progress=show_progress,
540545
)
541-
# TODO: Depending on decision trailing "/" needs to stay or not. EMS requires it to be present
542-
environment.build.path = uploaded_artifact.full_storage_path + "/"
546+
if environment.build is not None:
547+
# TODO: Depending on decision trailing "/" needs to stay or not. EMS requires it to be present
548+
environment.build.path = str(uploaded_artifact.full_storage_path) + "/"
543549
return environment
544550

545551

546-
def _get_snapshot_path_info(artifact) -> Tuple[Path, IgnoreFile, str]:
552+
def _get_snapshot_path_info(artifact) -> Optional[Tuple[Path, IgnoreFile, str]]:
547553
"""
548554
Validate an Artifact's local path and get its resolved path, ignore file, and hash. If no local path, return None.
549555
:param artifact: Artifact object

‎sdk/ml/azure-ai-ml/azure/ai/ml/_artifacts/_blob_storage_helper.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@
1010
import time
1111
import uuid
1212
from pathlib import Path, PurePosixPath
13-
from typing import TYPE_CHECKING, Dict, List, Optional, Union
13+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
1414

15-
from typing_extensions import Literal
1615
from colorama import Fore
16+
from typing_extensions import Literal
1717

1818
from azure.ai.ml._artifacts._constants import (
1919
ARTIFACT_ORIGIN,
@@ -23,6 +23,7 @@
2323
MAX_CONCURRENCY,
2424
UPLOAD_CONFIRMATION,
2525
)
26+
from azure.ai.ml._azure_environments import _get_cloud_details
2627
from azure.ai.ml._utils._asset_utils import (
2728
AssetNotChangedError,
2829
IgnoreFile,
@@ -32,7 +33,6 @@
3233
upload_directory,
3334
upload_file,
3435
)
35-
from azure.ai.ml._azure_environments import _get_cloud_details
3636
from azure.ai.ml.constants._common import STORAGE_AUTH_MISMATCH_ERROR
3737
from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MlException, ValidationException
3838
from azure.core.exceptions import ResourceNotFoundError
@@ -58,7 +58,7 @@ def __init__(self, credential: str, account_url: str, container_name: Optional[s
5858
self.total_file_count = 1
5959
self.uploaded_file_count = 0
6060
self.overwrite = False
61-
self.indicator_file = None
61+
self.indicator_file: Any = None
6262
self.legacy = False
6363
self.name = None
6464
self.version = None
@@ -142,12 +142,12 @@ def upload(
142142
time.sleep(0.5)
143143
self._set_confirmation_metadata(name, version)
144144
except AssetNotChangedError:
145-
name = self.name
146-
version = self.version
145+
name = str(self.name)
146+
version = str(self.version)
147147
if self.legacy:
148148
dest = dest.replace(ARTIFACT_ORIGIN, LEGACY_ARTIFACT_DIRECTORY)
149149

150-
artifact_info = {
150+
artifact_info: Dict = {
151151
"remote path": dest,
152152
"name": name,
153153
"version": version,

‎sdk/ml/azure-ai-ml/azure/ai/ml/_artifacts/_fileshare_storage_helper.py‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,8 @@
1010
import time
1111
from pathlib import Path, PurePosixPath
1212
from typing import Callable, Dict, Optional, Tuple, Union
13+
1314
from typing_extensions import Literal
14-
from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
15-
from azure.storage.fileshare import ShareDirectoryClient, ShareFileClient
1615

1716
from azure.ai.ml._artifacts._constants import (
1817
ARTIFACT_ORIGIN,
@@ -30,6 +29,8 @@
3029
get_upload_files_from_folder,
3130
)
3231
from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MlException
32+
from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
33+
from azure.storage.fileshare import ShareDirectoryClient, ShareFileClient
3334

3435
module_logger = logging.getLogger(__name__)
3536

@@ -123,11 +124,11 @@ def upload(
123124
time.sleep(0.5)
124125
self._set_confirmation_metadata(source, asset_id, name, version)
125126
else:
126-
name = self.name
127-
version = self.version
127+
name = str(self.name)
128+
version = str(self.version)
128129
if self.legacy:
129130
dest = dest.replace(ARTIFACT_ORIGIN, LEGACY_ARTIFACT_DIRECTORY)
130-
artifact_info = {"remote path": dest, "name": name, "version": version}
131+
artifact_info: Dict = {"remote path": dest, "name": name, "version": version}
131132

132133
return artifact_info
133134

@@ -165,19 +166,20 @@ def upload_file(
165166
with open(source, "rb") as data:
166167
if in_directory:
167168
file_name = dest.rsplit("/")[-1]
168-
if show_progress:
169-
subdirectory_client.upload_file(
170-
file_name=file_name,
171-
data=data,
172-
validate_content=validate_content,
173-
raw_response_hook=callback,
174-
)
175-
else:
176-
subdirectory_client.upload_file(
177-
file_name=file_name,
178-
data=data,
179-
validate_content=validate_content,
180-
)
169+
if subdirectory_client is not None:
170+
if show_progress:
171+
subdirectory_client.upload_file(
172+
file_name=file_name,
173+
data=data,
174+
validate_content=validate_content,
175+
raw_response_hook=callback,
176+
)
177+
else:
178+
subdirectory_client.upload_file(
179+
file_name=file_name,
180+
data=data,
181+
validate_content=validate_content,
182+
)
181183
else:
182184
if show_progress:
183185
with FileUploadProgressBar(msg=msg) as pbar:
@@ -220,12 +222,12 @@ def upload_dir(
220222
upload_paths = sorted(get_upload_files_from_folder(source_path, prefix=prefix, ignore_file=ignore_file))
221223
self.total_file_count = len(upload_paths)
222224

223-
for root, *_ in os.walk(source):
225+
for root, *_ in os.walk(source): # type: ignore[type-var]
224226
if sys.platform.startswith(("win32", "cygwin")):
225227
split_char = "\\"
226228
else:
227229
split_char = "/"
228-
trunc_root = root.rsplit(split_char)[-1]
230+
trunc_root = root.rsplit(split_char)[-1] # type: ignore[union-attr]
229231
subdir = subdir.create_subdirectory(trunc_root)
230232

231233
if show_progress:
@@ -321,7 +323,7 @@ def _get_asset_metadata(
321323
asset_id: str,
322324
default_items: Dict[str, bool],
323325
legacy_items: Dict[str, bool],
324-
) -> Tuple[Union[ShareDirectoryClient, ShareFileClient], Dict]:
326+
) -> Tuple:
325327
# if asset_id key's value doesn't match either bool,
326328
# it's not in the dictionary and we check "LocalUpload" dictionary below.
327329

‎sdk/ml/azure-ai-ml/azure/ai/ml/_artifacts/_gen2_storage_helper.py‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,12 @@
1111
import uuid
1212
from pathlib import Path, PurePosixPath
1313
from typing import Dict, List, Optional, Union
14-
from typing_extensions import Literal
14+
1515
from colorama import Fore
16+
from typing_extensions import Literal
1617

17-
from azure.ai.ml._artifacts._constants import UPLOAD_CONFIRMATION, FILE_SIZE_WARNING
18+
from azure.ai.ml._artifacts._constants import FILE_SIZE_WARNING, UPLOAD_CONFIRMATION
19+
from azure.ai.ml._azure_environments import _get_cloud_details
1820
from azure.ai.ml._utils._asset_utils import (
1921
AssetNotChangedError,
2022
IgnoreFile,
@@ -24,7 +26,6 @@
2426
upload_directory,
2527
upload_file,
2628
)
27-
from azure.ai.ml._azure_environments import _get_cloud_details
2829
from azure.ai.ml.constants._common import STORAGE_AUTH_MISMATCH_ERROR
2930
from azure.ai.ml.exceptions import ErrorCategory, ErrorTarget, MlException, ValidationException
3031
from azure.core.exceptions import ResourceExistsError
@@ -133,10 +134,10 @@ def upload(
133134
time.sleep(0.5)
134135
self._set_confirmation_metadata(name, version)
135136
except AssetNotChangedError:
136-
name = self.name
137-
version = self.version
137+
name = str(self.name)
138+
version = str(self.version)
138139

139-
artifact_info = {
140+
artifact_info: Dict = {
140141
"remote path": dest,
141142
"name": name,
142143
"version": version,
@@ -155,7 +156,7 @@ def check_blob_exists(self) -> None:
155156
overwritten with a complete upload.
156157
"""
157158
try:
158-
if self.directory_client.exists():
159+
if self.directory_client is not None and self.directory_client.exists():
159160
metadata = self.directory_client.get_directory_properties().metadata
160161

161162
if (
@@ -181,7 +182,8 @@ def check_blob_exists(self) -> None:
181182
raise e
182183

183184
def _set_confirmation_metadata(self, name: str, version: str) -> None:
184-
self.directory_client.set_metadata(_build_metadata_dict(name, version))
185+
if self.directory_client is not None:
186+
self.directory_client.set_metadata(_build_metadata_dict(name, version))
185187

186188
def download(self, starts_with: str, destination: Union[str, os.PathLike] = Path.home()) -> None:
187189
"""Downloads all items inside a specified filesystem directory with the prefix `starts_with` to the destination

‎sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_operations.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -938,7 +938,7 @@ def log_missing_uri(what: str) -> None:
938938
module_logger.info("Downloading artifact %s to %s", uri, destination)
939939
download_artifact_from_aml_uri(
940940
uri=uri,
941-
destination=destination,
941+
destination=destination, # type: ignore[arg-type]
942942
datastore_operation=self._datastore_operations,
943943
)
944944

‎sdk/ml/azure-ai-ml/azure/ai/ml/operations/_job_ops_helper.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ def stream_logs_until_completion(
276276
_current_logs_dict = (
277277
list_logs_in_datastore(
278278
ds_properties,
279-
prefix=prefix,
279+
prefix=str(prefix),
280280
legacy_log_folder_name=legacy_folder_name,
281281
)
282282
if ds_properties is not None

‎sdk/ml/azure-ai-ml/azure/ai/ml/operations/_model_operations.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ def create_or_update( # type: ignore
196196
body=get_asset_body_for_registry_storage(self._registry_name, "models", model.name, model.version),
197197
)
198198

199-
model, indicator_file = _check_and_upload_path(
199+
model, indicator_file = _check_and_upload_path( # type: ignore[type-var]
200200
artifact=model,
201201
asset_operations=self,
202202
sas_uri=sas_uri,

‎sdk/ml/azure-ai-ml/azure/ai/ml/operations/_operation_orchestrator.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ def _get_code_asset_arm_id(self, code_asset: Code, register_asset: bool = True)
287287
)
288288
uploaded_code_asset, _ = _check_and_upload_path(
289289
artifact=code_asset,
290-
asset_operations=self._code_assets,
290+
asset_operations=self._code_assets, # type: ignore[arg-type]
291291
artifact_type=ErrorTarget.CODE,
292292
show_progress=self._operation_config.show_progress,
293293
sas_uri=sas_info["sas_uri"],
@@ -318,7 +318,9 @@ def _get_environment_arm_id(self, environment: Environment, register_asset: bool
318318
env_response = self._environments.create_or_update(environment) # type: ignore[attr-defined]
319319
return env_response.id
320320
environment = _check_and_upload_env_build_context(
321-
environment=environment, operations=self._environments, show_progress=self._operation_config.show_progress
321+
environment=environment,
322+
operations=self._environments, # type: ignore[arg-type]
323+
show_progress=self._operation_config.show_progress,
322324
)
323325
environment._id = get_arm_id_with_version(
324326
self._operation_scope,
@@ -338,7 +340,7 @@ def _get_model_arm_id(self, model: Model, register_asset: bool = True) -> Union[
338340
return self._model.create_or_update(model).id # type: ignore[attr-defined]
339341
uploaded_model, _ = _check_and_upload_path(
340342
artifact=model,
341-
asset_operations=self._model,
343+
asset_operations=self._model, # type: ignore[arg-type]
342344
artifact_type=ErrorTarget.MODEL,
343345
show_progress=self._operation_config.show_progress,
344346
)
@@ -367,7 +369,7 @@ def _get_data_arm_id(self, data_asset: Data, register_asset: bool = True) -> Uni
367369
return self._data.create_or_update(data_asset).id # type: ignore[attr-defined]
368370
data_asset, _ = _check_and_upload_path(
369371
artifact=data_asset,
370-
asset_operations=self._data,
372+
asset_operations=self._data, # type: ignore[arg-type]
371373
artifact_type=ErrorTarget.DATA,
372374
show_progress=self._operation_config.show_progress,
373375
)

‎sdk/ml/azure-ai-ml/pyproject.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ exclude = [
2929
"tests",
3030
"setup.py",
3131
"samples",
32-
"azure/ai/ml/_artifacts/",
3332
"azure/ai/ml/_file_utils/",
3433
"azure/ai/ml/_local_endpoints/",
3534
"azure/ai/ml/_logging/",

0 commit comments

Comments
 (0)