fix downloading larger files with onedrive (#364)

Author: vangheem

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.4.5
+
+### Fixes
+
+* **Fix downloading large files for OneDrive**
+
 ## 0.4.4
 
 ### Fixes

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.4.4"  # pragma: no cover
+__version__ = "0.4.5"  # pragma: no cover

unstructured_ingest/v2/pipeline/pipeline.py

@@ -203,7 +203,14 @@ def apply_filter(self, records: list[dict]) -> list[dict]:
 
     def get_indices(self) -> list[dict]:
         if self.indexer_step.process.is_async():
-            indices = asyncio.run(self.indexer_step.run_async())
+
+            async def run_async():
+                output = []
+                async for i in self.indexer_step.run_async():
+                    output.append(i)
+                return output
+
+            indices = asyncio.run(run_async())
         else:
             indices = self.indexer_step.run()
         indices_inputs = [{"file_data_path": i} for i in indices]

unstructured_ingest/v2/processes/connectors/onedrive.py

@@ -223,7 +223,7 @@ class OnedriveDownloader(Downloader):
     download_config: OnedriveDownloaderConfig
 
     @SourceConnectionNetworkError.wrap
-    def _fetch_file(self, file_data: FileData):
+    def _fetch_file(self, file_data: FileData) -> DriveItem:
         if file_data.source_identifiers is None or not file_data.source_identifiers.fullpath:
             raise ValueError(
                 f"file data doesn't have enough information to get "
@@ -257,7 +257,7 @@ def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
                     file.download_session(f, chunk_size=1024 * 1024 * 100).execute_query()
             else:
                 with download_path.open(mode="wb") as f:
-                    file.download(f).execute_query()
+                    file.download_session(f).execute_query()
             return self.generate_download_response(file_data=file_data, download_path=download_path)
         except Exception as e:
             logger.error(f"[{CONNECTOR_TYPE}] Exception during downloading: {e}", exc_info=True)