Fix: Update examples to acknowledge actual folder structure; remove incorrect logging (#564)

For some reason all examples point to folders that are too high in the
hierarchy, making them always fail.
This PR changes them to point to actual example_data.

Another change is getting rid of logs that didn't even work - due to the
current logger approach, they've lacked a handler and never logged
anything.

Author: Chmielok

CHANGELOG.md

@@ -1,6 +1,10 @@
+## 1.1.1
+
+* **Fix: Update examples**
+
 ## 1.1.0
 
-- **Feature**: Embedding with OpenAI (or Azure OpenAI) can trust custom certificate authority by specifying environment variable REQUESTS_CA_BUNDLE.
+* **Feature**: Embedding with OpenAI (or Azure OpenAI) can trust custom certificate authority by specifying environment variable REQUESTS_CA_BUNDLE.
 
 ## 1.0.59
 

examples/airtable.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 
 from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.pipeline import Pipeline
 from unstructured_ingest.processes.chunker import ChunkerConfig
 from unstructured_ingest.processes.connectors.airtable import (
@@ -18,14 +17,13 @@
 from unstructured_ingest.processes.embedder import EmbedderConfig
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"
 download_path = work_dir / "download"
 
 if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
     Pipeline.from_configs(
         context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         indexer_config=AirtableIndexerConfig(

examples/azure_cognitive_search.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 
 from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.pipeline import Pipeline
 from unstructured_ingest.processes.chunker import ChunkerConfig
 from unstructured_ingest.processes.connectors.azure_ai_search import (
@@ -20,17 +19,16 @@
 from unstructured_ingest.processes.embedder import EmbedderConfig
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"
 download_path = work_dir / "download"
 
 if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
     index_name = "ingest-test-destination"
     Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         indexer_config=LocalIndexerConfig(
             input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
         ),

examples/chroma.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 
 from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.pipeline import Pipeline
 from unstructured_ingest.processes.chunker import ChunkerConfig
 from unstructured_ingest.processes.connectors.chroma import (
@@ -20,16 +19,15 @@
 from unstructured_ingest.processes.embedder import EmbedderConfig
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"
 download_path = work_dir / "download"
 
 if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
     Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         indexer_config=LocalIndexerConfig(input_path=docs_path.resolve() / "multisimple"),
         downloader_config=LocalDownloaderConfig(download_dir=download_path),
         source_connection_config=LocalConnectionConfig(),

examples/couchbase.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 
 from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.pipeline import Pipeline
 from unstructured_ingest.processes.chunker import ChunkerConfig
 from unstructured_ingest.processes.connectors.couchbase import (
@@ -19,16 +18,15 @@
 from unstructured_ingest.processes.embedder import EmbedderConfig
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"
 download_path = work_dir / "download"
 
 if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
     Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
         downloader_config=LocalDownloaderConfig(download_dir=download_path),
         source_connection_config=LocalConnectionConfig(),

examples/databricks_volumes_dest.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 
 from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.pipeline import Pipeline
 from unstructured_ingest.processes.chunker import ChunkerConfig
 from unstructured_ingest.processes.connectors.databricks.volumes_native import (
@@ -18,16 +17,15 @@
 )
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"
 download_path = work_dir / "download"
 
 if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
     Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/fake-text.txt"),
         downloader_config=LocalDownloaderConfig(download_dir=download_path),
         source_connection_config=LocalConnectionConfig(),

examples/databricks_volumes_source.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 
 from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.pipeline import Pipeline
 from unstructured_ingest.processes.chunker import ChunkerConfig
 from unstructured_ingest.processes.connectors.databricks.volumes_native import (
@@ -17,16 +16,15 @@
 )
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"
 download_path = work_dir / "download"
 
 if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
     Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         indexer_config=DatabricksNativeVolumesIndexerConfig(
             host=os.environ["DATABRICKS_HOST"],
             catalog=os.environ["DATABRICKS_CATALOG"],

examples/delta_table.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 
 from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.pipeline import Pipeline
 from unstructured_ingest.processes.chunker import ChunkerConfig
 from unstructured_ingest.processes.connectors.delta_table import (
@@ -19,16 +18,15 @@
 from unstructured_ingest.processes.embedder import EmbedderConfig
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"
 download_path = work_dir / "download"
 
 if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
     Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         indexer_config=LocalIndexerConfig(
             input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt",
         ),

examples/discord_example.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 
 from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.pipeline import Pipeline
 from unstructured_ingest.processes.connectors.discord import (
     CONNECTOR_TYPE,
@@ -14,14 +13,13 @@
 from unstructured_ingest.processes.connectors.local import LocalUploaderConfig
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"
 download_path = work_dir / "download"
 
 if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
     Pipeline.from_configs(
         context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
         indexer_config=DiscordIndexerConfig(channels=os.environ["DISCORD_CHANNELS"].split(",")),

examples/elasticsearch.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 
 from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.pipeline import Pipeline
 from unstructured_ingest.processes.chunker import ChunkerConfig
 from unstructured_ingest.processes.connectors.elasticsearch import (
@@ -20,17 +19,16 @@
 from unstructured_ingest.processes.embedder import EmbedderConfig
 from unstructured_ingest.processes.partitioner import PartitionerConfig
 
-base_path = Path(__file__).parent.parent.parent.parent
+base_path = Path(__file__).parent.parent
 docs_path = base_path / "example-docs"
 work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
 output_path = work_dir / "output"
 download_path = work_dir / "download"
 
 if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
     index_name = "ingest-test-destination"
     Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         indexer_config=LocalIndexerConfig(
             input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
         ),