feat: add --download-only parameter to unstructured-ingest (#416)

Add --download-only parameter so that files may be downloaded if they are not already present (as usual, in either --download-dir or the default download ~/.cache/... location if --download-dir is not specified) and skip processing them through unstructured.

Author: natygyoon

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.5.10-dev1
+## 0.5.10-dev2
 
 ### Enhancements
 
@@ -7,6 +7,8 @@
 
 ### Features
 
+* Added `--download-only` parameter to `unstructured-ingest`
+
 ### Fixes
 
 ## 0.5.9

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.10-dev1"  # pragma: no cover
+__version__ = "0.5.10-dev2"  # pragma: no cover

unstructured/ingest/connector/biomed.py

@@ -47,6 +47,7 @@ class SimpleBiomedConfig(BaseConnectorConfig):
     # where to write structured data, with the directory structure matching FTP path
     output_dir: str
     re_download: bool = False
+    download_only: bool = False
     preserve_downloads: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
@@ -142,7 +143,11 @@ def _output_filename(self):
         return Path(f"{self.file_meta.output_filepath}.json").resolve()
 
     def cleanup_file(self):
-        if not self.config.preserve_downloads and self.filename.is_file():
+        if (
+            not self.config.preserve_downloads
+            and self.filename.is_file()
+            and not self.config.download_only
+        ):
             logger.debug(f"Cleaning up {self}")
             Path.unlink(self.filename)
 
@@ -170,6 +175,8 @@ def get_file(self):
 
     def write_result(self):
         """Write the structured json result for this doc. result must be json serializable."""
+        if self.config.download_only:
+            return
         output_filename = self._output_filename()
         output_filename.parent.mkdir(parents=True, exist_ok=True)
         with open(output_filename, "w") as output_f:
@@ -182,7 +189,7 @@ class BiomedConnector(BaseConnector):
 
     def __init__(self, config):
         self.config = config
-        self.cleanup_files = not self.config.preserve_downloads
+        self.cleanup_files = not self.config.preserve_downloads and not self.config.download_only
 
     def _list_objects_api(self):
         def urls_to_metadata(urls):

unstructured/ingest/connector/fsspec.py

@@ -29,6 +29,7 @@ class SimpleFsspecConfig(BaseConnectorConfig):
     output_dir: str
     preserve_downloads: bool = False
     re_download: bool = False
+    download_only: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
     fields_include: str = "element_id,text,type,metadata"
@@ -121,6 +122,8 @@ def get_file(self):
 
     def write_result(self):
         """Write the structured json result for this doc. result must be json serializable."""
+        if self.config.download_only:
+            return
         output_filename = self._output_filename()
         output_filename.parent.mkdir(parents=True, exist_ok=True)
         with open(output_filename, "w") as output_f:
@@ -133,8 +136,8 @@ def filename(self):
         return self._tmp_download_file()
 
     def cleanup_file(self):
-        """Removes the local copy the file after successful processing."""
-        if not self.config.preserve_downloads:
+        """Removes the local copy of the file after successful processing."""
+        if not self.config.preserve_downloads and not self.config.download_only:
             logger.debug(f"Cleaning up {self}")
             os.unlink(self._tmp_download_file())
 
@@ -154,7 +157,7 @@ def __init__(
         self.fs: AbstractFileSystem = get_filesystem_class(self.config.protocol)(
             **self.config.access_kwargs,
         )
-        self.cleanup_files = not config.preserve_downloads
+        self.cleanup_files = not config.preserve_downloads and not config.download_only
 
     def cleanup(self, cur_dir=None):
         """cleanup linginering empty sub-dirs from s3 paths, but leave remaining files

unstructured/ingest/connector/git.py

@@ -26,6 +26,7 @@ class SimpleGitConfig(BaseConnectorConfig):
     output_dir: str
     preserve_downloads: bool = False
     re_download: bool = False
+    download_only: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
     fields_include: str = "element_id,text,type,metadata"
@@ -51,8 +52,8 @@ def _create_full_tmp_dir_path(self):
         self.filename.parent.mkdir(parents=True, exist_ok=True)
 
     def cleanup_file(self):
-        """Removes the local copy the file (or anything else) after successful processing."""
-        if not self.config.preserve_downloads:
+        """Removes the local copy of the file (or anything else) after successful processing."""
+        if not self.config.preserve_downloads and not self.config.download_only:
             logger.debug(f"Cleaning up {self}")
             os.unlink(self.filename)
 
@@ -76,6 +77,8 @@ def has_output(self):
 
     def write_result(self):
         """Write the structured json result for this doc. result must be json serializable."""
+        if self.config.download_only:
+            return
         output_filename = self._output_filename()
         output_filename.parent.mkdir(parents=True, exist_ok=True)
         with open(output_filename, "w", encoding="utf8") as output_f:
@@ -88,7 +91,7 @@ class GitConnector(BaseConnector):
     config: SimpleGitConfig
 
     def __post_init__(self) -> None:
-        self.cleanup_files = not self.config.preserve_downloads
+        self.cleanup_files = not self.config.preserve_downloads and not self.config.download_only
 
     def cleanup(self, cur_dir=None):
         if not self.cleanup_files:

unstructured/ingest/connector/google_drive.py

@@ -76,6 +76,7 @@ class SimpleGoogleDriveConfig(BaseConnectorConfig):
     # where to write structured data, with the directory structure matching drive path
     output_dir: str
     re_download: bool = False
+    download_only: bool = False
     preserve_downloads: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
@@ -106,7 +107,11 @@ def _output_filename(self):
         return Path(f"{self.file_meta.get('output_filepath')}.json").resolve()
 
     def cleanup_file(self):
-        if not self.config.preserve_downloads and self.filename.is_file():
+        if (
+            not self.config.preserve_downloads
+            and self.filename.is_file()
+            and not self.config.download_only
+        ):
             logger.debug(f"Cleaning up {self}")
             Path.unlink(self.filename)
 
@@ -174,6 +179,8 @@ def get_file(self):
 
     def write_result(self):
         """Write the structured json result for this doc. result must be json serializable."""
+        if self.config.download_only:
+            return
         output_filename = self._output_filename()
         output_filename.parent.mkdir(parents=True, exist_ok=True)
         with open(output_filename, "w") as output_f:
@@ -186,7 +193,7 @@ class GoogleDriveConnector(BaseConnector):
 
     def __init__(self, config):
         self.config = config
-        self.cleanup_files = not self.config.preserve_downloads
+        self.cleanup_files = not self.config.preserve_downloads and not self.config.download_only
 
     def _list_objects(self, drive_id, recursive=False):
         files = []

unstructured/ingest/connector/local.py

@@ -24,6 +24,7 @@ class SimpleLocalConfig(BaseConnectorConfig):
     file_glob: Optional[str] = None
 
     # base connector options
+    download_only: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
     fields_include: str = "element_id,text,type,metadata"
@@ -69,6 +70,8 @@ def has_output(self):
 
     def write_result(self):
         """Write the structured json result for this doc. result must be json serializable."""
+        if self.config.download_only:
+            return
         output_filename = self._output_filename()
         output_filename.parent.mkdir(parents=True, exist_ok=True)
         with open(output_filename, "w") as output_f:

unstructured/ingest/connector/reddit.py

@@ -31,6 +31,7 @@ class SimpleRedditConfig(BaseConnectorConfig):
     output_dir: str
     preserve_downloads: bool = False
     re_download: bool = False
+    download_only: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
     fields_include: str = "element_id,text,type,metadata"
@@ -57,8 +58,8 @@ def _create_full_tmp_dir_path(self):
         self.filename.parent.mkdir(parents=True, exist_ok=True)
 
     def cleanup_file(self):
-        """Removes the local copy the file (or anything else) after successful processing."""
-        if not self.config.preserve_downloads:
+        """Removes the local copy of the file (or anything else) after successful processing."""
+        if not self.config.preserve_downloads and not self.config.download_only:
             logger.debug(f"Cleaning up {self}")
             os.unlink(self.filename)
 
@@ -82,6 +83,8 @@ def has_output(self):
 
     def write_result(self):
         """Write the structured json result for this doc. result must be json serializable."""
+        if self.config.download_only:
+            return
         output_filename = self._output_filename()
         output_filename.parent.mkdir(parents=True, exist_ok=True)
         with open(output_filename, "w", encoding="utf8") as output_f:
@@ -100,7 +103,7 @@ def __init__(self, config: SimpleRedditConfig):
             client_secret=config.client_secret,
             user_agent=config.user_agent,
         )
-        self.cleanup_files = not config.preserve_downloads
+        self.cleanup_files = not config.preserve_downloads and not config.download_only
 
     def cleanup(self, cur_dir=None):
         if not self.cleanup_files:

unstructured/ingest/connector/wikipedia.py

@@ -26,6 +26,7 @@ class SimpleWikipediaConfig(BaseConnectorConfig):
     output_dir: str
     preserve_downloads: bool = False
     re_download: bool = False
+    download_only: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
     fields_include: str = "element_id,text,type,metadata"
@@ -52,8 +53,8 @@ def _create_full_tmp_dir_path(self):
         self.filename.parent.mkdir(parents=True, exist_ok=True)
 
     def cleanup_file(self):
-        """Removes the local copy the file (or anything else) after successful processing."""
-        if not self.config.preserve_downloads:
+        """Removes the local copy of the file (or anything else) after successful processing."""
+        if not self.config.preserve_downloads and not self.config.download_only:
             logger.debug(f"Cleaning up {self}")
             os.unlink(self.filename)
 
@@ -75,6 +76,8 @@ def has_output(self):
 
     def write_result(self):
         """Write the structured json result for this doc. result must be json serializable."""
+        if self.config.download_only:
+            return
         output_filename = self._output_filename()
         output_filename.parent.mkdir(parents=True, exist_ok=True)
         with open(output_filename, "w", encoding="utf8") as output_f:
@@ -133,7 +136,7 @@ def _output_filename(self):
 class WikipediaConnector(BaseConnector):
     def __init__(self, config: SimpleWikipediaConfig):
         self.config = config
-        self.cleanup_files = not config.preserve_downloads
+        self.cleanup_files = not config.preserve_downloads and not config.download_only
 
     def cleanup(self, cur_dir=None):
         if not self.cleanup_files:

unstructured/ingest/interfaces.py

@@ -48,6 +48,7 @@ class BaseConnectorConfig(ABC):
     # where to write structured data outputs
     output_dir: str
     re_download: bool = False
+    download_only: bool = False
     metadata_include: Optional[str] = None
     metadata_exclude: Optional[str] = None
     fields_include: str = "element_id,text,type,metadata"
@@ -93,8 +94,9 @@ def write_result(self):
         pass
 
     def process_file(self):
+        if self.config.download_only:
+            return
         logger.info(f"Processing {self.filename}")
-
         elements = partition(filename=str(self.filename))
         isd_elems = convert_to_dict(elements)