Merge pull request #57 from Cosmo-Tech/LAL/dataset_api_v5-PROD-14582

Lal/dataset api v5 prod 14582

Author: neomatamune

cosmotech/coal/cosmotech_api/dataset/download/file.py

@@ -118,7 +118,7 @@ def process_txt(target_file) -> Dict[str, Any]:
     LOGGER.debug(T("coal.services.dataset.processing_text").format(file_name=target_file))
     with open(target_file, "r") as _file:
         current_filename = os.path.basename(target_file)
-        content[current_filename] = "\n".join(line for line in _file)
+        content[current_filename] = "".join(line for line in _file)
 
         line_count = content[current_filename].count("\n") + 1
         LOGGER.debug(
@@ -128,7 +128,7 @@ def process_txt(target_file) -> Dict[str, Any]:
 
 
 def read_file(file_name, file):
-    @timed(f"process{file_name}", debug=True)
+    @timed(f"process {file_name}", debug=True)
     def timed_read_file(file_name, file):
         content = {}
         if ".xls" in file_name:

cosmotech/coal/cosmotech_api/runner/datasets.py

@@ -63,9 +63,9 @@ def download_dataset(
 
     csm_version = semver_of("cosmotech_api")
     if csm_version.major >= 5:
-        download_dataset_v5(organization_id, workspace_id, dataset_id, read_files)
+        return download_dataset_v5(organization_id, workspace_id, dataset_id, read_files)
     else:
-        download_dataset_v4(organization_id, workspace_id, dataset_id, read_files)
+        return download_dataset_v4(organization_id, workspace_id, dataset_id, read_files)
 
 
 def download_dataset_v5(
@@ -94,6 +94,7 @@ def download_dataset_v5(
                                                    workspace_id=workspace_id,
                                                    dataset_id=dataset_id)
 
+        content = dict()
         tmp_dataset_dir = tempfile.mkdtemp()
         tmp_dataset_dir_path = Path(tmp_dataset_dir)
         for part in dataset.parts:
@@ -103,10 +104,10 @@ def download_dataset_v5(
                 binary_file.write(data_part)
 
             if read_files:
-                content = file.read_file(part_file_path)
+                content.update(file.read_file(part.source_name, part_file_path))
 
         return {
-            "type": "twincache",
+            "type": "csm_dataset",
             "content": content,
             "name": dataset.name,
             "folder_path": str(part_file_path),

cosmotech/coal/utils/decorator.py

@@ -11,12 +11,13 @@ def decorator(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
             process_start = time.time()
-            func(*args, **kwargs)
+            r = func(*args, **kwargs)
             process_time = time.time() - process_start
             msg = T("coal.common.timing.operation_completed").format(operation=operation, time=process_time)
             if debug:
                 LOGGER.debug(msg)
             else:
                 LOGGER.info(msg)
+            return r
         return wrapper
     return decorator

cosmotech/coal/utils/semver.py

@@ -0,0 +1,6 @@
+from packaging.version import Version
+import importlib.metadata
+
+
+def semver_of(package: str) -> Version:
+    return Version(importlib.metadata.version(package))