Fixes issue #1071

Author: Your Name

sde_collections/tasks.py

@@ -144,129 +144,6 @@ def resolve_title_pattern(title_pattern_id):
     title_pattern = TitlePattern.objects.get(id=title_pattern_id)
     title_pattern.apply()
 
-
-"""
-@celery_app.task
-def fetch_and_update_full_text(collection_id):
-
-    try:
-        collection = Collection.objects.get(id=collection_id)
-    except Collection.DoesNotExist:
-        raise Exception(f"Collection with ID {collection_id} does not exist.")
-
-    url = "https://sde-lrm.nasa-impact.net/api/v1/engine.sql" #LRM_DEV Server
-    sql_command = f"SELECT url1, text, title FROM sde_index WHERE collection = '/SDE/{collection.config_folder}/'"
-    token = os.getenv('LRMDEV_TOKEN')
-
-
-    payload = json.dumps({
-        "method": "engine.sql",
-        "sql": sql_command,
-        "pretty": True,
-        "log": False,
-        "output": "json",
-        "resolveIndexList": "false",
-        "engines": "default"
-    })
-
-    headers = {
-        'Content-Type': 'application/json',
-        'Authorization': f'Bearer {token}'
-    }
-
-    response = requests.post(url, headers=headers, data=payload)
-    if response.status_code == 200:
-        records = response.json().get("Rows", [])
-        for record in records:
-            url, full_text, title = record
-            if not url or not full_text or not title:
-                continue
-            # Directly update or create the entry without checking for content changes
-            CandidateURL.objects.update_or_create(
-                url=url,
-                collection=collection,
-                defaults={
-                    'scraped_text': full_text,
-                    'scraped_title': title
-                }
-            )
-
-        return f"Processed {len(records)} records; Updated or created in database."
-    else:
-        raise Exception(f"Failed to fetch text: {response.status_code} {response.text}")
-    """
-
-# You will have to have a different function for Li's server as it uses user and pw with body to login.
-# If the sinequa web token is used, can user&pw be removed from the body? if yes then can integrate, but headers will b diff (auth/cookie). if lis then header1, elif lrm_dev then h2, else h3
-# Fill in the tokens in the .django file
-
-# Integrated - LRM devs and Lis separate
-"""
-@celery_app.task
-def fetch_and_update_full_text(collection_id, server_type):
-    try:
-        collection = Collection.objects.get(id=collection_id)
-    except Collection.DoesNotExist:
-        raise Exception(f"Collection with ID {collection_id} does not exist.")
-
-    # Server-specific configurations
-    server_config = get_server_config(server_type)
-
-    # API Request Parameters
-    payload = json.dumps({
-        "method": "engine.sql",
-        "sql": f"SELECT url1, text, title FROM sde_index WHERE collection = '/SDE/{collection.config_folder}/'",
-        "pretty": True,
-        "log": False,
-        "output": "json",
-        "resolveIndexList": "false",
-        "engines": "default"
-    })
-
-    token = server_config["token"]
-    url = server_config["url"]
-    headers = {
-        'Content-Type': 'application/json',
-        'Authorization': f'Bearer {token}'
-    }
-
-    # Send the request
-    response = requests.post(url, headers=headers, data=payload)
-    if response.status_code == 200:
-        records = response.json().get("Rows", [])
-        for record in records:
-            url, full_text, title = record
-            if not url or not full_text or not title:
-                continue
-            CandidateURL.objects.update_or_create(
-                url=url,
-                collection=collection,
-                defaults={
-                    'scraped_text': full_text,
-                    'scraped_title': title
-                }
-            )
-        return f"Processed {len(records)} records; Updated or created in database."
-    else:
-        raise Exception(f"Failed to fetch text: {response.status_code} {response.text}")
-
-
-def get_server_config(server_type):
-    if server_type == "LRM_DEV":
-        return {
-            "url": "https://sde-lrm.nasa-impact.net/api/v1/engine.sql",
-            "token": os.getenv("LRMDEV_TOKEN")
-        }
-    elif server_type == "LIS":
-        return {
-            "url": "http://sde-xli.nasa-impact.net/api/v1/engine.sql",
-            "token": os.getenv("LIS_TOKEN")
-        }
-    else:
-        raise ValueError("Invalid server type.")
-"""
-
-
 @celery_app.task
 def fetch_and_update_full_text(collection_id, server_type):
     try:
@@ -321,3 +198,4 @@ def get_server_config(server_type):
         return {"url": "http://sde-xli.nasa-impact.net/api/v1/engine.sql", "token": os.getenv("LIS_TOKEN")}
     else:
         raise ValueError("Invalid server type.")
+