bugfix/fix sql download file data (#313)

* require source identifier content on file data, fix batch connectors

* bump changelog

* add utility to get either file data types from dict

* create solid release

* populate source identifier in mock file data

* regenerate other fixtures

* tidy

Author: rbiseck3

CHANGELOG.md

@@ -1,9 +1,13 @@
-## 0.3.11-dev0
+## 0.3.11
 
 ### Enhancements
 
 * **Support Databricks personal access token**
 
+### Fixes
+
+* **Fix missing source identifiers in some downloaders**
+
 ## 0.3.10
 
 ### Enhancements

test/integration/connectors/expected_results/astradb/file_data/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv.json

@@ -1,7 +1,11 @@
 {
   "identifier": "25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv",
   "connector_type": "astradb",
-  "source_identifiers": null,
+  "source_identifiers": {
+    "filename": "25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv",
+    "fullpath": "25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv",
+    "rel_path": null
+  },
   "metadata": {
     "url": null,
     "version": null,
@@ -10,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734108082.652201",
+    "date_processed": "1734616909.789815",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -19,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-75/test_astra_search_source0/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv",
   "display_name": null
 }

test/integration/connectors/expected_results/astradb/file_data/43d02113-723f-5ec1-acbb-c8da8d3650dc.json

@@ -8,7 +8,7 @@
     "record_locator": null,
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734108082.497229",
+    "date_processed": "1734616909.6431518",
     "permissions_data": null,
     "filesize_bytes": null
   },

test/integration/connectors/expected_results/astradb/file_data/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv.json

@@ -1,7 +1,11 @@
 {
   "identifier": "60297eea-73d7-4fca-a97e-ea73d7cfca62.csv",
   "connector_type": "astradb",
-  "source_identifiers": null,
+  "source_identifiers": {
+    "filename": "60297eea-73d7-4fca-a97e-ea73d7cfca62.csv",
+    "fullpath": "60297eea-73d7-4fca-a97e-ea73d7cfca62.csv",
+    "rel_path": null
+  },
   "metadata": {
     "url": null,
     "version": null,
@@ -10,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734108082.6517608",
+    "date_processed": "1734616909.789366",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -19,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-75/test_astra_search_source0/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv",
   "display_name": null
 }

test/integration/connectors/expected_results/astradb/file_data/641d99e3-9941-4c18-9d99-e399414c183d.csv.json

@@ -1,7 +1,11 @@
 {
   "identifier": "641d99e3-9941-4c18-9d99-e399414c183d.csv",
   "connector_type": "astradb",
-  "source_identifiers": null,
+  "source_identifiers": {
+    "filename": "641d99e3-9941-4c18-9d99-e399414c183d.csv",
+    "fullpath": "641d99e3-9941-4c18-9d99-e399414c183d.csv",
+    "rel_path": null
+  },
   "metadata": {
     "url": null,
     "version": null,
@@ -10,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734108082.652407",
+    "date_processed": "1734616909.788801",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -19,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-75/test_astra_search_source0/641d99e3-9941-4c18-9d99-e399414c183d.csv",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/641d99e3-9941-4c18-9d99-e399414c183d.csv",
   "display_name": null
 }

test/integration/connectors/expected_results/astradb/file_data/762c0093-2277-4f3e-ac00-932277af3e0e.csv.json

@@ -1,7 +1,11 @@
 {
   "identifier": "762c0093-2277-4f3e-ac00-932277af3e0e.csv",
   "connector_type": "astradb",
-  "source_identifiers": null,
+  "source_identifiers": {
+    "filename": "762c0093-2277-4f3e-ac00-932277af3e0e.csv",
+    "fullpath": "762c0093-2277-4f3e-ac00-932277af3e0e.csv",
+    "rel_path": null
+  },
   "metadata": {
     "url": null,
     "version": null,
@@ -10,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734108082.651469",
+    "date_processed": "1734616909.789599",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -19,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-75/test_astra_search_source0/762c0093-2277-4f3e-ac00-932277af3e0e.csv",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/762c0093-2277-4f3e-ac00-932277af3e0e.csv",
   "display_name": null
 }

test/integration/connectors/expected_results/astradb/file_data/ae40df94-0b3a-4f89-80df-940b3a6f8966.csv.json

@@ -1,7 +1,11 @@
 {
   "identifier": "ae40df94-0b3a-4f89-80df-940b3a6f8966.csv",
   "connector_type": "astradb",
-  "source_identifiers": null,
+  "source_identifiers": {
+    "filename": "ae40df94-0b3a-4f89-80df-940b3a6f8966.csv",
+    "fullpath": "ae40df94-0b3a-4f89-80df-940b3a6f8966.csv",
+    "rel_path": null
+  },
   "metadata": {
     "url": null,
     "version": null,
@@ -10,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734108082.651985",
+    "date_processed": "1734616909.789103",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -19,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-75/test_astra_search_source0/ae40df94-0b3a-4f89-80df-940b3a6f8966.csv",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/pytest-of-romanisecke/pytest-0/test_astra_search_source0/ae40df94-0b3a-4f89-80df-940b3a6f8966.csv",
   "display_name": null
 }

test/integration/connectors/expected_results/mongodb/file_data/172fdbf9-275b-5e38-82e6-4ec730d42460.json

@@ -11,7 +11,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1734380792.896338",
+    "date_processed": "1734616944.92011",
     "permissions_data": null,
     "filesize_bytes": null
   },

test/integration/connectors/expected_results/mongodb/file_data/659daefa21dd8c9054b084b6.json

@@ -4,7 +4,7 @@
   "source_identifiers": {
     "filename": "659daefa21dd8c9054b084b6.txt",
     "fullpath": "659daefa21dd8c9054b084b6.txt",
-    "rel_path": "659daefa21dd8c9054b084b6.txt"
+    "rel_path": null
   },
   "metadata": {
     "url": null,
@@ -16,7 +16,7 @@
     },
     "date_created": "2024-01-09T20:39:22+00:00",
     "date_modified": null,
-    "date_processed": "1734380792.896338",
+    "date_processed": "1734616944.92011",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -25,6 +25,6 @@
     "collection": "sample-mongodb-data"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmpyfmsos4r/659daefa21dd8c9054b084b6.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmpc2l6l7w3/659daefa21dd8c9054b084b6.txt",
   "display_name": null
 }

test/integration/connectors/expected_results/mongodb/file_data/659daefa21dd8c9054b084b7.json

@@ -4,7 +4,7 @@
   "source_identifiers": {
     "filename": "659daefa21dd8c9054b084b7.txt",
     "fullpath": "659daefa21dd8c9054b084b7.txt",
-    "rel_path": "659daefa21dd8c9054b084b7.txt"
+    "rel_path": null
   },
   "metadata": {
     "url": null,
@@ -16,7 +16,7 @@
     },
     "date_created": "2024-01-09T20:39:22+00:00",
     "date_modified": null,
-    "date_processed": "1734380792.896338",
+    "date_processed": "1734616944.92011",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -25,6 +25,6 @@
     "collection": "sample-mongodb-data"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmpyfmsos4r/659daefa21dd8c9054b084b7.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmpc2l6l7w3/659daefa21dd8c9054b084b7.txt",
   "display_name": null
 }