change to a cleaner output for api.get_full_texts

Author: CarsonDavis

sde_collections/sinequa_api.py

@@ -162,26 +162,23 @@ def get_full_texts(self, collection_config_folder: str, source: str = None) -> A
         Retrieves the full texts, URLs, and titles for a specified collection.
 
         Returns:
-            dict: A JSON response containing the results of the SQL query in an expected format under the 'Rows' key,
-                where each item has 'url1', 'text', and 'title' .
+            dict: A JSON response containing the results of the SQL query,
+                where each item has 'url', 'text', and 'title'.
 
         Example:
             Calling get_full_texts("example_collection") might return:
-            {
-                'Rows': [
+                [
                     {
-                        'url1': 'http://example.com/article1',
+                        'url': 'http://example.com/article1',
                         'text': 'Here is the full text of the first article...',
                         'title': 'Article One Title'
                     },
                     {
-                        'url1': 'http://example.com/article2',
+                        'url': 'http://example.com/article2',
                         'text': 'Here is the full text of the second article...',
                         'title': 'Article Two Title'
                     }
                 ]
-            }
-
         """
 
         if not source:
@@ -191,4 +188,11 @@ def get_full_texts(self, collection_config_folder: str, source: str = None) -> A
             raise ValueError("Index not defined for this server")
 
         sql = f"SELECT url1, text, title FROM {index} WHERE collection = '/{source}/{collection_config_folder}/'"
-        return self.sql_query(sql)
+        full_text_response = self.sql_query(sql)
+        return self._process_full_text_response(full_text_response)
+
+    @staticmethod
+    def _process_full_text_response(full_text_response: str):
+        return [
+            {"url": url, "full_text": full_text, "title": title} for url, full_text, title in full_text_response["Rows"]
+        ]

sde_collections/tasks.py

@@ -160,18 +160,16 @@ def fetch_and_update_full_text(collection_id, server_name):
     """
     collection = Collection.objects.get(id=collection_id)
     api = Api(server_name)
-    full_texts = api.get_full_texts(collection.config_folder)
+    documents = api.get_full_texts(collection.config_folder)
 
-    records = full_texts.get("Rows", [])
-    if not records:
-        return "No records found in the response."
-
-    for record in records:
-        url, full_text, title = record
-        if not (url and full_text and title):
+    for doc in documents:
+        # if all values are not present, then it is skipped?
+        if not (doc["url"] and doc["full_text"] and doc["title"]):
             continue
 
         CandidateURL.objects.update_or_create(
-            url=url, collection=collection, defaults={"scraped_text": full_text, "scraped_title": title}
+            url=doc["url"],
+            collection=collection,
+            defaults={"scraped_text": doc["full_text"], "scraped_title": doc["title"]},
         )
-    return f"Successfully processed {len(records)} records and updated the database."
+    return f"Successfully processed {len(documents)} records and updated the database."