MOD: Batch download now returns a list of paths

Author: nmacholl

databento/historical/api/batch.py

@@ -233,7 +233,7 @@ def download(
         job_id: str,
         filename_to_download: Optional[str] = None,
         enable_partial_downloads: bool = True,
-    ) -> None:
+    ) -> List[Path]:
         """
         Download a batch job or a specific file to `{output_dir}/{job_id}/`.
 
@@ -254,6 +254,18 @@ def download(
         enable_partial_downloads : bool, default True
             If partially downloaded files will be resumed using range request(s).
 
+        Returns
+        -------
+        List[Path]
+            A list of paths to the downloaded files.
+
+        Raises
+        ------
+        RuntimeError
+            When no files were found for the batch job.
+        ValueError
+            When a file fails to download.
+
         """
         output_dir = validate_path(output_dir, "output_dir")
         self._check_api_key()
@@ -270,7 +282,7 @@ def download(
 
         if not job_files:
             logger.error("Cannot download batch job %s (no files found).", job_id)
-            return
+            raise RuntimeError(f"no files for batch job {job_id}")
 
         if filename_to_download:
             # A specific file is being requested
@@ -287,12 +299,15 @@ def download(
                     job_id,
                     filename_to_download,
                 )
-                return
+                raise ValueError(
+                    f"{filename_to_download} is not a file for batch job {job_id}",
+                )
 
         # Prepare job directory
         job_dir = Path(output_dir) / job_id
         os.makedirs(job_dir, exist_ok=True)
 
+        file_paths = []
         for details in job_files:
             filename = str(details["filename"])
             output_path = job_dir / filename
@@ -320,6 +335,9 @@ def download(
                 output_path=output_path,
                 enable_partial_downloads=enable_partial_downloads,
             )
+            file_paths.append(output_path)
+
+        return file_paths
 
     def _download_file(
         self,
@@ -355,7 +373,7 @@ async def download_async(
         job_id: str,
         filename_to_download: Optional[str] = None,
         enable_partial_downloads: bool = True,
-    ) -> None:
+    ) -> List[Path]:
         """
         Asynchronously download a batch job or a specific file to
         `{output_dir}/{job_id}/`.
@@ -377,6 +395,18 @@ async def download_async(
         enable_partial_downloads : bool, default True
             If partially downloaded files will be resumed using range request(s).
 
+        Returns
+        -------
+        List[Path]
+            A list of paths to the downloaded files.
+
+        Raises
+        ------
+        RuntimeError
+            When no files were found for the batch job.
+        ValueError
+            When a file fails to download.
+
         """
         output_dir = validate_path(output_dir, "output_dir")
         self._check_api_key()
@@ -393,7 +423,7 @@ async def download_async(
 
         if not job_files:
             logger.error("Cannot download batch job %s (no files found).", job_id)
-            return
+            raise RuntimeError(f"no files for batch job {job_id}")
 
         if filename_to_download:
             # A specific file is being requested
@@ -406,16 +436,19 @@ async def download_async(
                     break
             if not is_file_found:
                 logger.error(
-                    "Cannot download batch job %s file " "(%s not found)",
+                    "Cannot download batch job %s file (%s not found)",
                     job_id,
                     filename_to_download,
                 )
-                return
+                raise ValueError(
+                    f"{filename_to_download} is not a file for batch job {job_id}",
+                )
 
         # Prepare job directory
         job_dir = Path(output_dir) / job_id
         os.makedirs(job_dir, exist_ok=True)
 
+        file_paths = []
         for details in job_files:
             filename = str(details["filename"])
             output_path = job_dir / filename
@@ -443,6 +476,9 @@ async def download_async(
                 output_path=output_path,
                 enable_partial_downloads=enable_partial_downloads,
             )
+            file_paths.append(output_path)
+
+        return file_paths
 
     async def _download_file_async(
         self,

examples/historical_batch_download.py

@@ -6,7 +6,10 @@
     client = Historical(key=key)
 
     # Will download all job files to a `my_data/YOUR_JOB_ID/` directory
-    client.batch.download(
+    downloaded_files = client.batch.download(
         output_dir="my_data",
         job_id="YOUR_JOB_ID",  # <-- Discover this from `.list_jobs(...)`
     )
+
+    for file in downloaded_files:
+        print(f"Downloaded {file.name}")

examples/historical_batch_download_async.py

@@ -8,11 +8,14 @@ async def example_download_batch_job_async() -> None:
     client = Historical(key=key)
 
     # Will download all job files to a `my_data/YOUR_JOB_ID/` directory
-    await client.batch.download_async(
+    downloaded_files = await client.batch.download_async(
         output_dir="my_data",
         job_id="YOUR_JOB_ID",  # <-- Discover this from `.list_jobs(...)`
     )
 
+    for file in downloaded_files:
+        print(f"Downloaded {file.name}")
+
 
 if __name__ == "__main__":
     asyncio.run(example_download_batch_job_async())

tests/test_historical_batch.py

@@ -170,11 +170,13 @@ def test_batch_download_single_file_sends_expected_request(
         filename = "glbx-mdp3-20220610.mbo.csv.zst"
 
         # Act
-        self.client.batch.download(
-            job_id=job_id,
-            output_dir="my_data",
-            filename_to_download=filename,
-        )
+        with pytest.raises(ValueError):
+            # We expect this to fail since this is not a real batch job.
+            self.client.batch.download(
+                job_id=job_id,
+                output_dir="my_data",
+                filename_to_download=filename,
+            )
 
         # Assert
         call = mocked_get.call_args.kwargs