Skip to content

Commit 4bdc882

Browse files
committed
Remove redundant self.spec attribute from ExternalTable
Access spec via self.storage.spec instead of storing it as a separate attribute. StorageBackend already stores the spec internally.
1 parent 5a727d2 commit 4bdc882

File tree

1 file changed

+15
-13
lines changed

1 file changed

+15
-13
lines changed

src/datajoint/external.py

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ class ExternalTable(Table):
3838

3939
def __init__(self, connection, store, database):
4040
self.store = store
41-
self.spec = config.get_store_spec(store)
4241
self.database = database
4342
self._connection = connection
4443
self._heading = Heading(
@@ -53,7 +52,7 @@ def __init__(self, connection, store, database):
5352
if not self.is_declared:
5453
self.declare()
5554
# Initialize storage backend (validates configuration)
56-
self.storage = StorageBackend(self.spec)
55+
self.storage = StorageBackend(config.get_store_spec(store))
5756

5857
@property
5958
def definition(self):
@@ -84,36 +83,37 @@ def s3(self):
8483
from . import s3
8584

8685
if not hasattr(self, "_s3_legacy") or self._s3_legacy is None:
87-
self._s3_legacy = s3.Folder(**self.spec)
86+
self._s3_legacy = s3.Folder(**self.storage.spec)
8887
return self._s3_legacy
8988

9089
# - low-level operations - private
9190

9291
def _make_external_filepath(self, relative_filepath):
9392
"""resolve the complete external path based on the relative path"""
93+
spec = self.storage.spec
9494
# Strip root for S3 paths
95-
if self.spec["protocol"] == "s3":
96-
posix_path = PurePosixPath(PureWindowsPath(self.spec["location"]))
95+
if spec["protocol"] == "s3":
96+
posix_path = PurePosixPath(PureWindowsPath(spec["location"]))
9797
location_path = (
9898
Path(*posix_path.parts[1:])
99-
if len(self.spec["location"]) > 0 and any(case in posix_path.parts[0] for case in ("\\", ":"))
99+
if len(spec["location"]) > 0 and any(case in posix_path.parts[0] for case in ("\\", ":"))
100100
else Path(posix_path)
101101
)
102102
return PurePosixPath(location_path, relative_filepath)
103103
# Preserve root for local filesystem
104-
elif self.spec["protocol"] == "file":
105-
return PurePosixPath(Path(self.spec["location"]), relative_filepath)
104+
elif spec["protocol"] == "file":
105+
return PurePosixPath(Path(spec["location"]), relative_filepath)
106106
else:
107107
# For other protocols (gcs, azure, etc.), treat like S3
108-
location = self.spec.get("location", "")
108+
location = spec.get("location", "")
109109
return PurePosixPath(location, relative_filepath) if location else PurePosixPath(relative_filepath)
110110

111111
def _make_uuid_path(self, uuid, suffix=""):
112112
"""create external path based on the uuid hash"""
113113
return self._make_external_filepath(
114114
PurePosixPath(
115115
self.database,
116-
"/".join(subfold(uuid.hex, self.spec["subfolding"])),
116+
"/".join(subfold(uuid.hex, self.storage.spec["subfolding"])),
117117
uuid.hex,
118118
).with_suffix(suffix)
119119
)
@@ -235,9 +235,11 @@ def upload_filepath(self, local_filepath):
235235
"""
236236
local_filepath = Path(local_filepath)
237237
try:
238-
relative_filepath = str(local_filepath.relative_to(self.spec["stage"]).as_posix())
238+
relative_filepath = str(local_filepath.relative_to(self.storage.spec["stage"]).as_posix())
239239
except ValueError:
240-
raise DataJointError("The path {path} is not in stage {stage}".format(path=local_filepath.parent, **self.spec))
240+
raise DataJointError(
241+
f"The path {local_filepath.parent} is not in stage {self.storage.spec['stage']}"
242+
)
241243
uuid = uuid_from_buffer(init_string=relative_filepath) # hash relative path, not contents
242244
contents_hash = uuid_from_file(local_filepath)
243245

@@ -285,7 +287,7 @@ def _need_checksum(local_filepath, expected_size):
285287
"filepath", "contents_hash", "size"
286288
)
287289
external_path = self._make_external_filepath(relative_filepath)
288-
local_filepath = Path(self.spec["stage"]).absolute() / relative_filepath
290+
local_filepath = Path(self.storage.spec["stage"]).absolute() / relative_filepath
289291

290292
file_exists = Path(local_filepath).is_file() and (
291293
not _need_checksum(local_filepath, size) or uuid_from_file(local_filepath) == contents_hash

0 commit comments

Comments
 (0)