Do not output orig_elements for astradb: original_elements has the correctly truncated field (#435)

vangheem · web-flow · commit ec84155bd004 · 2025-03-20T19:06:41.000-04:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.5.17
+
+### Fixes 
+
+* **Do not output `orig_elements` for astradb** `original_elements` has the correctly truncated field
+
 ## 0.5.16
 
 ### Fixes 
diff --git a/test/integration/connectors/expected_results/astradb/file_data/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv.json b/test/integration/connectors/expected_results/astradb/file_data/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv.json
@@ -14,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734616909.789815",
+    "date_processed": "1742508009.2832398",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -23,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv",
+  "local_download_path": "/private/var/folders/7s/nsbls8k12qngtdvkhhkpf1n00000gn/T/pytest-of-nathan/pytest-325/test_astra_search_source0/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv",
   "display_name": null
 }
diff --git a/test/integration/connectors/expected_results/astradb/file_data/43d02113-723f-5ec1-acbb-c8da8d3650dc.json b/test/integration/connectors/expected_results/astradb/file_data/43d02113-723f-5ec1-acbb-c8da8d3650dc.json
@@ -8,7 +8,7 @@
     "record_locator": null,
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734616909.6431518",
+    "date_processed": "1742508009.054028",
     "permissions_data": null,
     "filesize_bytes": null
   },
diff --git a/test/integration/connectors/expected_results/astradb/file_data/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv.json b/test/integration/connectors/expected_results/astradb/file_data/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv.json
@@ -14,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734616909.789366",
+    "date_processed": "1742508009.284526",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -23,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv",
+  "local_download_path": "/private/var/folders/7s/nsbls8k12qngtdvkhhkpf1n00000gn/T/pytest-of-nathan/pytest-325/test_astra_search_source0/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv",
   "display_name": null
 }
diff --git a/test/integration/connectors/expected_results/astradb/file_data/641d99e3-9941-4c18-9d99-e399414c183d.csv.json b/test/integration/connectors/expected_results/astradb/file_data/641d99e3-9941-4c18-9d99-e399414c183d.csv.json
@@ -14,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734616909.788801",
+    "date_processed": "1742508009.284161",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -23,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/641d99e3-9941-4c18-9d99-e399414c183d.csv",
+  "local_download_path": "/private/var/folders/7s/nsbls8k12qngtdvkhhkpf1n00000gn/T/pytest-of-nathan/pytest-325/test_astra_search_source0/641d99e3-9941-4c18-9d99-e399414c183d.csv",
   "display_name": null
 }
diff --git a/test/integration/connectors/expected_results/astradb/file_data/762c0093-2277-4f3e-ac00-932277af3e0e.csv.json b/test/integration/connectors/expected_results/astradb/file_data/762c0093-2277-4f3e-ac00-932277af3e0e.csv.json
@@ -14,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734616909.789599",
+    "date_processed": "1742508009.283753",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -23,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/762c0093-2277-4f3e-ac00-932277af3e0e.csv",
+  "local_download_path": "/private/var/folders/7s/nsbls8k12qngtdvkhhkpf1n00000gn/T/pytest-of-nathan/pytest-325/test_astra_search_source0/762c0093-2277-4f3e-ac00-932277af3e0e.csv",
   "display_name": null
 }
diff --git a/test/integration/connectors/expected_results/astradb/file_data/ae40df94-0b3a-4f89-80df-940b3a6f8966.csv.json b/test/integration/connectors/expected_results/astradb/file_data/ae40df94-0b3a-4f89-80df-940b3a6f8966.csv.json
@@ -14,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734616909.789103",
+    "date_processed": "1742508009.284849",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -23,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/ae40df94-0b3a-4f89-80df-940b3a6f8966.csv",
+  "local_download_path": "/private/var/folders/7s/nsbls8k12qngtdvkhhkpf1n00000gn/T/pytest-of-nathan/pytest-325/test_astra_search_source0/ae40df94-0b3a-4f89-80df-940b3a6f8966.csv",
   "display_name": null
 }
diff --git a/test/integration/connectors/expected_results/astradb/stager/DA-1p-with-duplicate-pages.pdf.json b/test/integration/connectors/expected_results/astradb/stager/DA-1p-with-duplicate-pages.pdf.json
diff --git a/test/integration/connectors/expected_results/astradb/stager/DA-1p-with-duplicate-pages.pdf.ndjson b/test/integration/connectors/expected_results/astradb/stager/DA-1p-with-duplicate-pages.pdf.ndjson
diff --git a/test/integration/connectors/expected_results/astradb/stager_flatten_metadata/DA-1p-with-duplicate-pages.pdf.json b/test/integration/connectors/expected_results/astradb/stager_flatten_metadata/DA-1p-with-duplicate-pages.pdf.json
diff --git a/test/integration/connectors/expected_results/astradb/stager_flatten_metadata/DA-1p-with-duplicate-pages.pdf.ndjson b/test/integration/connectors/expected_results/astradb/stager_flatten_metadata/DA-1p-with-duplicate-pages.pdf.ndjson
diff --git a/unstructured_ingest/__version__.py b/unstructured_ingest/__version__.py
@@ -1 +1 @@
-__version__ = "0.5.16"  # pragma: no cover
+__version__ = "0.5.17"  # pragma: no cover
diff --git a/unstructured_ingest/v2/processes/connectors/astradb.py b/unstructured_ingest/v2/processes/connectors/astradb.py
@@ -112,7 +112,6 @@ def get_astra_collection(
     collection_name: str,
     keyspace: str,
 ) -> "AstraDBCollection":
-
     astra_db = get_astra_db(connection_config=connection_config, keyspace=keyspace)
 
     # astradb will return a collection object in all cases (even if it doesn't exist)
@@ -315,6 +314,7 @@ def truncate_dict_elements(self, element_dict: dict) -> None:
                     text_as_html, MAX_CONTENT_PARAM_BYTE_SIZE
                 )
             metadata["original_elements"] = format_and_truncate_orig_elements(element_dict)
+            metadata.pop("orig_elements", None)
 
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         self.truncate_dict_elements(element_dict)
