feat/autogenerate click options (#20)

* Migrate to autogenerated click options from the pydantic Base Models

* Add changelog entry

* tidy

* Add in azure cognitive search

* Fix optional bool mapping

* Remove unneeded cli config files

* Minor fixes

Author: rbiseck3

.github/workflows/e2e.yml

@@ -29,8 +29,7 @@ jobs:
   test_src:
     strategy:
       matrix:
-#        python-version: ["3.9","3.10"]
-        python-version: [ "3.10" ]
+        python-version: ["3.9","3.10"]
     runs-on: ubuntu-latest-m
     needs: [ setup ]
     steps:

CHANGELOG.md

@@ -1,9 +1,10 @@
-## 0.0.4-dev1
+## 0.0.4-dev2
 
 ### Enhancements
 
 * **Add Couchbase Destination Connector** Adds support for storing artifacts in Couchbase DB for Vector Search
 * **Leverage pydantic base models** All user-supplied configs are now derived from pydantic base models to leverage better type checking and add built in support for sensitive fields.
+* **Autogenerate click options from base models** Leverage th pydantic base models for all configs to autogenerate teh cli options exposed when running ingest as a CLI.
 
 ## 0.0.3
 

test_e2e/dest/azure-cognitive-search.sh

@@ -85,7 +85,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --chunk-new-after-n-chars 1500 \
   --chunk-max-characters 2500 \
   --chunk-multipage-sections \
-  --chunk-no-include-orig-elements \
+  --no-chunk-include-orig-elements \
   --embedding-provider "langchain-huggingface" \
   azure-cognitive-search \
   --key "$AZURE_SEARCH_API_KEY" \

test_e2e/env_setup/couchbase/common/check_cluster_health.py

@@ -15,6 +15,7 @@ class ClusterConfig:
 
 
 def check_bucket_health(cluster_config: ClusterConfig, url: str):
+    print("checking bucket health")
     max_attempts = 20
     attempt = 0
 
@@ -51,6 +52,7 @@ def check_bucket_health(cluster_config: ClusterConfig, url: str):
 
 
 def check_fts_service_health(cluster_config: ClusterConfig, url: str):
+    print("Checking FTS service health")
     max_attempts = 20
     attempt = 0
 

test_e2e/env_setup/couchbase/destination_connector/ingest_destination_setup_cluster.py

@@ -152,3 +152,4 @@ def setup_cluster(cluster_config: ClusterConfig):
     )
 
     setup_cluster(config)
+    print("Done provisioning couchbase cluster.")

test_e2e/python/test-ingest-couchbase-output.py

@@ -17,6 +17,7 @@ def get_client(username, password, connection_string) -> Cluster:
     auth = PasswordAuthenticator(username, password)
     options = ClusterOptions(auth)
     options.apply_profile("wan_development")
+    print(f"Creating client to {connection_string} with options {options}")
     cluster = Cluster(connection_string, options)
     cluster.wait_until_ready(timedelta(seconds=5))
     return cluster
@@ -66,17 +67,23 @@ def check(ctx, expected_docs):
     scope_name = ctx.parent.params["scope"]
     collection_name = ctx.parent.params["collection"]
 
+    print(
+        f"Checking that the number of docs match expected "
+        f"at {bucket_name}.{scope_name}.{collection_name}: {expected_docs}"
+    )
     # Tally up the embeddings
     query_result = cluster.query(f"Select * from {bucket_name}.{scope_name}.{collection_name}")
     docs = list(query_result)
     number_of_docs = len(docs)
 
     # Check that the assertion is true
     assert number_of_docs == expected_docs, (
-        f"Number of rows in generated table ({number_of_docs})"
+        f"Number of rows in generated table ({number_of_docs}) "
         f"doesn't match expected value: {expected_docs}"
     )
 
+    print("Number of docs matched expected")
+
 
 @cli.command()
 @click.option("--output-json", type=click.File())
@@ -87,7 +94,7 @@ def check_vector(ctx, output_json):
     exact_embedding = json_content[0][key_0]["embedding"]
     exact_text = json_content[0][key_0]["text"]
 
-    print("exact embedding:", len(exact_embedding), exact_embedding)
+    print("embedding length:", len(exact_embedding))
 
     cluster: Cluster = ctx.obj["cluster"]
     bucket_name = ctx.parent.params["bucket"]
@@ -132,6 +139,9 @@ def check_vector(ctx, output_json):
         assert not math.isclose(rows[1].score, 1, abs_tol=1e-4)
         assert rows[1].fields["text"] != exact_text
 
+    print("Embeddings check passed")
+
 
 if __name__ == "__main__":
+    print("Validating results")
     cli()

test_e2e/src/google-drive.sh

@@ -47,7 +47,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --output-dir "$OUTPUT_DIR" \
   --verbose \
   --drive-id 1OQZ66OHBE30rNsNa7dweGLfRmXvkT_jr \
-  --service-account-key "$GCP_INGEST_SERVICE_KEY_FILE" \
+  --service-account-key-path "$GCP_INGEST_SERVICE_KEY_FILE" \
   --recursive \
   --extensions "pdf,docx" \
   --work-dir "$WORK_DIR"

test_e2e/src/local-single-file-chunk-no-orig-elements.sh

@@ -38,9 +38,9 @@ RUN_SCRIPT=${RUN_SCRIPT:-./unstructured_ingest/main.py}
 PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   local \
   --chunking-strategy by_title \
-  --chunk-no-include-orig-elements \
+  --no-chunk-include-orig-elements \
   --chunk-max-characters 2000 \
-  --chunk-no-multipage-sections \
+  --no-chunk-multipage-sections \
   --input-path "$ABS_INPUT_PATH" \
   --metadata-exclude coordinates,filename,file_directory,metadata.data_source.date_created,metadata.data_source.date_modified,metadata.data_source.date_processed,metadata.last_modified,metadata.detection_class_prob,metadata.parent_id,metadata.category_depth \
   --num-processes "$max_processes" \

test_e2e/src/salesforce.sh

@@ -50,7 +50,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --download-dir "$DOWNLOAD_DIR" \
   --username "$SALESFORCE_USERNAME" \
   --consumer-key "$SALESFORCE_CONSUMER_KEY" \
-  --private-key "$SALESFORCE_PRIVATE_KEY_PATH" \
+  --private-key-path "$SALESFORCE_PRIVATE_KEY_PATH" \
   --metadata-exclude coordinates,filename,file_directory,metadata.data_source.date_processed,metadata.last_modified,metadata.detection_class_prob,metadata.parent_id,metadata.category_depth \
   --num-processes "$max_processes" \
   --preserve-downloads \

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.0.4-dev1"  # pragma: no cover
+__version__ = "0.0.4-dev2"  # pragma: no cover