Merge pull request #927 from NASA-IMPACT/926-improve-indexing-scripts

926 improve indexing scripts

Author: CarsonDavis

config_generation/config_example.py

@@ -1,60 +1,49 @@
-# rename this file to config.py and do not track it on git
-# example on how to get the token are in the README.md
-# go down to the bottom to the section called this is the stuff you want to change
+from sources_to_scrape import (
+    sources_to_index_test_grid_20240809,
+)
 
-from sources_to_scrape import sources_to_delete_20231108
-
-# API Config
 tokens: dict[str, str] = {
     "test_server": "token here",
     "ren_server": "token here",
 }
 
 AVAILABLE_INDEXERS_TEST = [
-    "NodeIndexer1/identity0",
-    "NodeWebapp1/identity0",
-    "NodeWebapp2/identity0",
-    "NodeWebapp3/identity0",
+    "IndexerServerA/identity0",
+    "IndexerServerB/identity0",
 ]
 
 AVAILABLE_INDEXERS_PROD = ["NodeINDEX1/identity0", "NodeINDEX2/identity0"]
 
 TEST_SERVER_INDEXES = [  # this is the test server list
-    "HELIO_Repository_1",
-    "SDE_Acronyms",
-    "SMD_ASTRO_Repository_1",
-    "SMD_ASTRO_Repository_2",
-    "SMD_EARTHSCIENCE_Repository_1",
-    "SMD_GENELAB_Repository_1",
-    "SMD_PLANETARY_Repository_1",
-    "SMD_PLANETARY_Repository_2",
+    # "sde_neural_test_index",
+    "sde_index"
 ]
 
 PROD_SERVER_INDEXES = [
     # "EDP_Audit_1",
-    "SMD_LSDA_Repository_1",
-    # "EDP_UserMetadata_1",
-    "SMD_NTRS_Repository_1",
-    "GCMD_Repository_1",
-    "SMD_PLANETARY_Repository_1",
-    # "GCMD_Repository_1_Metadata",
-    "SMD_PLANETARY_Repository_2",
-    "GCMD_Repository_2",
-    "STI_Repository_1",
-    # "GCMD_Repository_3_Metadata",
-    # "STI_Repository_1_Metadata",
-    "HELIO_Repository_1",
-    "STI_Repository_2",
+    # "SMD_LSDA_Repository_1",
+    # # "EDP_UserMetadata_1",
+    # "SMD_NTRS_Repository_1",
+    # "GCMD_Repository_1",
+    # "SMD_PLANETARY_Repository_1",
+    # # "GCMD_Repository_1_Metadata",
+    # "SMD_PLANETARY_Repository_2",
+    # "GCMD_Repository_2",
+    # "STI_Repository_1",
+    # # "GCMD_Repository_3_Metadata",
+    # # "STI_Repository_1_Metadata",
+    # "HELIO_Repository_1",
+    # "STI_Repository_2",
     "SDE_Index",
     # "STI_Repository_2_Metadata",
-    "SMD_ASTRO_Repository_1",
-    "STI_Repository_3",
-    "SMD_ASTRO_Repository_2",
-    "STI_Repository_4",
-    "SMD_EARTHSCIENCE_Repository_1",
-    # "SinequaDoc",
-    "SMD_GENELAB_Repository_1",
-    "Test",
+    # "SMD_ASTRO_Repository_1",
+    # "STI_Repository_3",
+    # "SMD_ASTRO_Repository_2",
+    # "STI_Repository_4",
+    # "SMD_EARTHSCIENCE_Repository_1",
+    # # "SinequaDoc",
+    # "SMD_GENELAB_Repository_1",
+    # "Test",
 ]
 
 
@@ -69,12 +58,14 @@
     },
 }
 
-# this is the stuff you want to change
-collection_list: list[str] = sources_to_delete_20231108  # python list
-source = "SMD"
-batch_name: str = "delete_everywhere_20231108_test"
+# Job Creation Config
+collection_list: list[str] = sources_to_index_test_grid_20240809  # python list
+date = "20240809"
+source = "SDE"
 server = "test"
 
 # auto assigned
+batch_delete_name: str = f"sources_to_delete_on_{server}_{date}"
+batch_index_name: str = f"sources_to_index_on_{server}_{date}"
 available_indexers = SERVER_INFO[server]["indexers"]
 indexes_to_delete_from = SERVER_INFO[server]["indexes"]

config_generation/delete_server_content.py

@@ -2,7 +2,7 @@
 
 from db_to_xml_file_based import XmlEditor
 
-from config import batch_name, collection_list, indexes_to_delete_from, source
+from config import batch_delete_name, collection_list, indexes_to_delete_from, source
 
 COMMAND_FILES_PATH = "../sinequa_configs/commands/"
 DELETE_COMMAND_TEMPLATE_PATH = "xmls/delete_template.xml"
@@ -13,5 +13,5 @@
     for index in indexes_to_delete_from:
         sql = f"delete from {index} where collection='/{source}/{collection}/'"
         command_file.update_or_add_element_value(element_name="SQL", element_value=sql, add_duplicate=True)
-file_name = f"{COMMAND_FILES_PATH}{batch_name}.xml"
+file_name = f"{COMMAND_FILES_PATH}{batch_delete_name}.xml"
 command_file._update_config_xml(file_name)

config_generation/generate_jobs.py

@@ -7,7 +7,7 @@
 
 from db_to_xml_file_based import XmlEditor
 
-from config import available_indexers, batch_name, collection_list, source
+from config import available_indexers, batch_index_name, collection_list, source
 
 
 class ParallelJobCreator:
@@ -36,7 +36,10 @@ def _create_job_name(self, collection_name):
         this code generates that file name as a string, and it will be passed to the function that
             creates the actual job file
         """
-        return f"collection.indexer.{collection_name}.xml"
+        if source == "SDE":
+            return f"collection.indexer.{collection_name}.xml"
+        else:
+            return f"collection.indexer.{source}.{collection_name}.xml"
 
     def _create_joblist_name(self, index):
         """
@@ -46,7 +49,7 @@ def _create_joblist_name(self, index):
         this code generates that file name as a string, and it will be passed to the function that
             creates the actual job file
         """
-        return f"parallel_indexing_list-{batch_name}-{index}.xml"
+        return f"parallel_indexing_list-{batch_index_name}-{index}.xml"
 
     def _create_collection_jobs(self):
         """
@@ -89,7 +92,7 @@ def make_all_parallel_jobs(self):
         master = XmlEditor(self.joblist_template_path)
         master.update_or_add_element_value("RunJobsInParallel", "true")
         [master.add_job_list_item(job_name) for job_name in job_names]
-        master._update_config_xml(f"{self.job_path_root}parallel_indexing_list-{batch_name}-master.xml")
+        master._update_config_xml(f"{self.job_path_root}parallel_indexing_list-{batch_index_name}-master.xml")
 
 
 if __name__ == "__main__":