Merge branch '1014-add-logs-when-importing-urls-so-we-know-how-many-were-expected-how-many-succeeded-and-how-many-failed' of https://github.com/NASA-IMPACT/COSMOS into 1014-add-logs-when-importing-urls-so-we-know-how-many-were-expected-how-many-succeeded-and-how-many-failed

Author: Your Name

CHANGELOG.md

@@ -36,10 +36,18 @@ For each PR made, an entry should be added to this changelog. It should contain
     - Used regex to catch any HTML content comming in as an input to form fields
     - Called this class within the serializer for necessary fields
 
+
 - 1014-add-logs-when-importing-urls-so-we-know-how-many-were-expected-how-many-succeeded-and-how-many-failed
-  - Description: When URLs of a given collection are imported into COSMOS, a Slack notification is sent. This notification includes the total count of the existing curated URLs, URLs imported from the server, dump URLs imported, delta URLs identified and delta URLs marked for deletion.
+  - Description: When URLs of a given collection are imported into COSMOS, a Slack notification is sent. This notification includes the name of the collection imported,count of the existing curated URLs, total URLs count as per the server, URLs successfully imported from the server, delta URLs identified and delta URLs marked for deletion.
   - Changes:
-    - Updated collection class to include the following functions: count_curated_urls(),count_dump_urls(),count_delta_urls(),count_marked_for_deletion_urls()
-    - fetch_and_replace_full_text() triggers send_detailed_import_notification()
+    - The get_full_texts() function in sde_collections/sinequa_api.py is updated to yeild total_count along with rows.
+    - fetch_and_replace_full_text() function in sde_collections/tasks.py captures the total_server_count and triggers send_detailed_import_notification().
     - Added a function send_detailed_import_notification() in sde_collections/utils/slack_utils.py to structure the notification to be sent.
     - Updated the associated tests effected due to inclusion of this functionality.
+
+- 3227-bugfix-title-patterns-selecting-multi-url-pattern-does-nothing
+  - Description: When selecting options from the match pattern type filter, the system does not filter the results as expected. Instead of displaying only the chosen variety of patterns, it continues to show all patterns.
+  - Changes:
+    - In `title_patterns_table` definition, corrected the column reference
+    - Made `match_pattern_type` searchable
+    - Corrected the column references and made code consistent on all the other tables, i.e., `exclude_patterns_table`, `include_patterns_table`, `division_patterns_table` and `document_type_patterns_table`

sde_collections/sinequa_api.py

@@ -257,7 +257,7 @@ def get_full_texts(
                 if total_count is None:
                     total_count = response.get("TotalRowCount", 0)
 
-                yield (self._process_rows_to_records(rows),total_count)
+                yield (self._process_rows_to_records(rows), total_count)
 
                 current_offset += len(rows)
 

sde_collections/tasks.py

@@ -16,7 +16,7 @@
 )
 from sde_collections.utils import slack_utils
 
-from .models.delta_url import DumpUrl,DeltaUrl,CuratedUrl
+from .models.delta_url import CuratedUrl, DeltaUrl, DumpUrl
 from .sinequa_api import Api
 from .utils.github_helper import GitHubHandler
 
@@ -174,11 +174,11 @@ def fetch_and_replace_full_text(collection_id, server_name):
     # Step 1: Delete existing DumpUrl entries
     deleted_count, _ = DumpUrl.objects.filter(collection=collection).delete()
     print(f"Deleted {deleted_count} old records.")
-    total_server_count=0
+    total_server_count = 0
     try:
         # Step 2: Process data in batches
         total_processed = 0
-        for batch,total_count in api.get_full_texts(collection.config_folder):
+        for batch, total_count in api.get_full_texts(collection.config_folder):
             if total_server_count == 0:
                 total_server_count = total_count
             with transaction.atomic():

sde_indexing_helper/static/js/delta_url_list.js

@@ -578,12 +578,10 @@ function initializeDataTable() {
             $("#exclude-patterns-dropdown-1").prop("disabled", true);
           } else if (index === 1) {
             $("#exclude-patterns-dropdown-1").on("change", function () {
-              if ($(this).val() === "") table.columns(6).search("").draw();
+              if ($(this).val() === "") table.columns(7).search("").draw();
               else {
-                table
-                  .column(6)
-                  .search(matchPatternTypeMap[$(this).val()])
-                  .draw();
+                const patternType = matchPatternTypeMap[$(this).val()];
+                table.column(7).search(patternType).draw();
               }
             });
           }
@@ -621,7 +619,7 @@ function initializeDataTable() {
         },
       },
       { data: "id", visible: false, searchable: false },
-      { data: "match_pattern_type", visible: false },
+      { data: "match_pattern_type", visible: false, searchable: true },
     ],
   });
 
@@ -671,11 +669,11 @@ function initializeDataTable() {
           } else {
             if (index === 1) {
               $("#include-patterns-dropdown-1").on("change", function () {
-                if ($(this).val() === "") table.columns(5).search("").draw();
-                table
-                  .column(5)
-                  .search(matchPatternTypeMap[$(this).val()])
-                  .draw();
+                if ($(this).val() === "") table.columns(6).search("").draw();
+                else {
+                  const patternType = matchPatternTypeMap[$(this).val()];
+                  table.column(6).search(patternType).draw();
+                }
               });
             }
           }
@@ -707,7 +705,7 @@ function initializeDataTable() {
         },
       },
       { data: "id", visible: false, searchable: false },
-      { data: "match_pattern_type", visible: false },
+      { data: "match_pattern_type", visible: false, searchable: true },
     ],
   });
 
@@ -754,12 +752,10 @@ function initializeDataTable() {
             $("#title-patterns-dropdown-1").prop("disabled", true);
           } else if (index === 1) {
             $("#title-patterns-dropdown-1").on("change", function () {
-              if ($(this).val() === "") table.columns(6).search("").draw();
+              if ($(this).val() === "") table.columns(7).search("").draw();
               else {
-                table
-                  .column(6)
-                  .search(matchPatternTypeMap[$(this).val()])
-                  .draw();
+                const patternType = matchPatternTypeMap[$(this).val()];
+                table.column(7).search(patternType).draw();
               }
             });
           }
@@ -792,7 +788,7 @@ function initializeDataTable() {
         },
       },
       { data: "id", visible: false, searchable: false },
-      { data: "match_pattern_type", visible: false },
+      { data: "match_pattern_type", visible: false, searchable: true },
     ],
   });
 
@@ -833,53 +829,19 @@ function initializeDataTable() {
     pageLength: 100,
     ajax: `/api/document-type-patterns/?format=datatables&collection_id=${collection_id}`,
     initComplete: function (data) {
-      this.api()
-        .columns()
-        .every(function (index) {
-          var table = $("#document_type_patterns_table").DataTable();
-
-          let addDropdownSelect = {
-            1: {
-              columnToSearch: 6,
-              matchPattern: {
-                "Individual URL Pattern": 1,
-                "Multi-URL Pattern": 2,
-              },
-            },
-            2: {
-              columnToSearch: 7,
-              matchPattern: {
-                Images: 1,
-                Data: 2,
-                Documentation: 3,
-                "Software and Tools": 4,
-                "Missions and Instruments": 5,
-              },
-            },
-          };
-
-          let column = this;
-          if (column.data().length === 0) {
-            $(`#document-type-patterns-dropdown-${index}`).prop(
-              "disabled",
-              true
-            );
-          } else if (index in addDropdownSelect) {
-            $("#document-type-patterns-dropdown-" + index).on(
-              "change",
-              function () {
-                let col = addDropdownSelect[index].columnToSearch;
-                let searchInput =
-                  addDropdownSelect[index].matchPattern[$(this).val()];
-                if ($(this).val() === "" || $(this).val() === undefined)
-                  table.columns(col).search("").draw();
-                else {
-                  table.columns(col).search(searchInput).draw();
-                }
-              }
-            );
-          }
-        });
+      var table = $("#document_type_patterns_table").DataTable();
+      this.api().columns().every(function (index) {
+        if (index === 1) {
+          $("#document-type-patterns-dropdown-1").on("change", function () {
+            if ($(this).val() === "") {
+              table.column(7).search("").draw();
+            } else {
+              const patternType = matchPatternTypeMap[$(this).val()];
+              table.column(7).search(patternType).draw();
+            }
+          });
+        }
+      });
     },
 
     columns: [
@@ -909,7 +871,7 @@ function initializeDataTable() {
         },
       },
       { data: "id", visible: false, searchable: false },
-      { data: "match_pattern_type", visible: false },
+      { data: "match_pattern_type", visible: false, searchable: true },
       { data: "document_type", visible: false },
     ],
   });
@@ -945,47 +907,20 @@ var division_patterns_table = $("#division_patterns_table").DataTable({
   pageLength: 100,
   ajax: `/api/division-patterns/?format=datatables&collection_id=${collection_id}`,
   initComplete: function (data) {
-    this.api()
-      .columns()
-      .every(function (index) {
-        var table = $("#division_patterns_table").DataTable();
-
-        let addDropdownSelect = {
-          1: {
-            columnToSearch: 6,
-            matchPattern: {
-              "Individual URL Pattern": 1,
-              "Multi-URL Pattern": 2,
-            },
-          },
-          2: {
-            columnToSearch: 7,
-            matchPattern: {
-              "Astrophysics": 1,
-              "Biological and Physical Sciences": 2,
-              "Earth Science": 3,
-              "Heliophysics": 4,
-              "Planetary Science": 5,
-            },
-          },
-        };
-
-        let column = this;
-        if (column.data().length === 0) {
-          $(`#division-patterns-dropdown-${index}`).prop("disabled", true);
-        } else if (index in addDropdownSelect) {
-          $("#division-patterns-dropdown-" + index).on("change", function () {
-            let col = addDropdownSelect[index].columnToSearch;
-            let searchInput =
-              addDropdownSelect[index].matchPattern[$(this).val()];
-            if ($(this).val() === "" || $(this).val() === undefined)
-              table.columns(col).search("").draw();
-            else {
-              table.columns(col).search(searchInput).draw();
-            }
-          });
-        }
-      });
+    var table = $("#division_patterns_table").DataTable();
+    this.api().columns().every(function (index) {
+      if (index === 1) {
+        // Updated pattern type dropdown handler
+        $("#division-patterns-dropdown-1").on("change", function () {
+          if ($(this).val() === "") {
+            table.column(7).search("").draw();
+          } else {
+            const patternType = matchPatternTypeMap[$(this).val()];
+            table.column(7).search(patternType).draw();
+          }
+        });
+      }
+    });
   },
 
   columns: [
@@ -1015,7 +950,7 @@ var division_patterns_table = $("#division_patterns_table").DataTable({
       },
     },
     { data: "id", visible: false, searchable: false },
-    { data: "match_pattern_type", visible: false },
+    { data: "match_pattern_type", visible: false, searchable: true },
     { data: "division", visible: false },
   ],
 });