Merge branch 'dev' into 1182-ml-classification-queue

Author: CarsonDavis

CHANGELOG.md

@@ -12,9 +12,13 @@ For each PR made, an entry should be added to this changelog. It should contain
   - etc.
 
 ## Changelog
-- 1182-ml-classification-queue
+
+- 1209-bug-fix-document-type-creator-form
+  - Description: The dropdown on the pattern creation form needs to be set as multi as the default option since this is why the doc type creator form is used for the majority of multi-URL pattern creations. This should be applied to doc types, division types, and titles as well.
   - Changes:
-    - a new env value has been created called `INFERENCE_API_URL`
+    - Set the default value for `match_pattern_type` in `BaseMatchPattern` class is set to `2`
+    - Changed `test_create_simple_exclude_pattern` test within `TestDeltaExcludePatternBasics`
+    - Changed `test_create_division_pattern` and `test_create_document_type_pattern_single` within `TestFieldModifierPatternBasics`
 
 - 1052-update-cosmos-to-create-jobs-for-scrapers-and-indexers
   - Description: The original automation set up to generate the scrapers and indexers automatically based on a collection workflow status change needed to be updated to more accurately reflect the curation workflow. It would also be good to generate the jobs during this process to streamline the same.
@@ -104,3 +108,32 @@ For each PR made, an entry should be added to this changelog. It should contain
     - Added universal search functionality tests
     - Created search pane filter tests
     - Added pattern application form tests with validation checks
+
+- 1101-bug-fix-quotes-not-escaped-in-titles
+  - Description: Title rules that include single quotes show up correctly in the sinequa frontend (and the COSMOS api) but not in the delta urls page.
+  - Changes:
+    - Added `escapeHtml` function in the `delta_url_list.js` file to handle special character escaping correctly.
+    - Called this function while retrieving the titles in `getGeneratedTitleColumn()` and `getCuratedGeneratedTitleColumn()` functions.
+
+- 1240-fix-code-scanning-alert-inclusion-of-functionality-from-an-untrusted-source
+  - Description: Ensured all external resources load securely by switching to HTTPS and adding Subresource Integrity (SRI) checks.
+  - Changes:
+    - Replaced protocol‑relative URLs with HTTPS.
+    - Added SRI (integrity) and crossorigin attributes to external script tags.
+
+- 1196-arrange-the-show-100-csv-customize-columns-boxes-to-be-in-one-line-on-the-delta-urls-page
+  changelog-update-Issue-1001
+  - Description: Formatting the buttons - 'Show 100','CSV' and 'Customize Columns' to be on a single line for an optimal use of space.
+  - Changes:
+    - Updated delta_url_list.css and delta_url_list.js files with necessary modifications
+
+- 1246-minor-enhancement-document-type-pattern-form-require-document-type-or-show-appropriate-error
+  - Description: In the Document Type Pattern Form, if the user does not select a Document Type while filling out the form, an appropriate error message is displayed.
+  - Changes:
+    - Added a JavaScript validation check on form submission to ensure the document type (stored in a hidden input) is not empty.
+    - Display an error message and prevent form submission if the field is empty.
+
+- 1249-add-https-link-to-cors_allowed_origins-for-sde-lrm
+  - Description: The feedback form API was throwing CORS errors and to rectify that, we need to add the apt https link for sde-lrm.
+  - Changes:
+    - Added `https://sde-lrm.nasa-impact.net` to `CORS_ALLOWED_ORIGINS` in the base settings.

config/settings/base.py

@@ -93,6 +93,7 @@
 CORS_ALLOWED_ORIGINS = [
     "http://localhost:3000",
     "http://sde-lrm.nasa-impact.net",
+    "https://sde-lrm.nasa-impact.net",
     "https://sde-qa.nasa-impact.net",
     "https://sciencediscoveryengine.test.nasa.gov",
     "https://sciencediscoveryengine.nasa.gov",

sde_collections/models/delta_patterns.py

@@ -30,7 +30,7 @@ class MatchPatternTypeChoices(models.IntegerChoices):
     match_pattern = models.CharField(
         "Pattern", help_text="This pattern is compared against the URL of all documents in the collection"
     )
-    match_pattern_type = models.IntegerField(choices=MatchPatternTypeChoices.choices, default=1)
+    match_pattern_type = models.IntegerField(choices=MatchPatternTypeChoices.choices, default=2)
     delta_urls = models.ManyToManyField(
         "DeltaUrl",
         related_name="%(class)ss",  # Makes delta_url.deltaincludepatterns.all()

sde_collections/tests/test_exclude_patterns.py

@@ -56,7 +56,7 @@ def test_create_simple_exclude_pattern(self):
         pattern = DeltaExcludePattern.objects.create(
             collection=self.collection, match_pattern="https://example.com/exclude-me", reason="Test exclusion"
         )
-        assert pattern.match_pattern_type == DeltaExcludePattern.MatchPatternTypeChoices.INDIVIDUAL_URL
+        assert pattern.match_pattern_type == DeltaExcludePattern.MatchPatternTypeChoices.MULTI_URL_PATTERN
 
     def test_exclude_single_curated_url(self):
         """Test excluding a single curated URL creates appropriate delta."""

sde_collections/tests/test_field_modifier_patterns.py

@@ -47,6 +47,7 @@ def test_create_document_type_pattern_single(self):
             collection=self.collection,
             match_pattern="https://example.com/docs/guide.pdf",
             document_type=DocumentTypes.DOCUMENTATION,
+            match_pattern_type=DeltaDocumentTypePattern.MatchPatternTypeChoices.INDIVIDUAL_URL,
         )
         assert pattern.match_pattern_type == DeltaDocumentTypePattern.MatchPatternTypeChoices.INDIVIDUAL_URL
         assert pattern.document_type == DocumentTypes.DOCUMENTATION
@@ -68,6 +69,7 @@ def test_create_division_pattern(self):
             collection=self.collection,
             match_pattern="https://example.com/helio/data.html",
             division=Divisions.HELIOPHYSICS,
+            match_pattern_type=DeltaDivisionPattern.MatchPatternTypeChoices.INDIVIDUAL_URL,
         )
         assert pattern.match_pattern_type == DeltaDivisionPattern.MatchPatternTypeChoices.INDIVIDUAL_URL
         assert pattern.division == Divisions.HELIOPHYSICS

sde_indexing_helper/static/css/delta_url_list.css

@@ -211,8 +211,6 @@
 }
 
 .modalFooter {
-    position: sticky;
-    bottom: 0;
     position: sticky;
     bottom: 0;
     padding: 10px 0;
@@ -255,8 +253,6 @@
     font-weight: 500;
 }
 
-
-
 .custom-select,
 .buttons-csv,
 .customizeColumns,
@@ -440,7 +436,6 @@ div.dt-buttons .btn.processing:after {
 }
 
 
-/* pagination position */
 div.dt-container div.dt-paging ul.pagination {
     position: absolute;
     right: 60px;
@@ -451,3 +446,31 @@ div.dt-container div.dt-paging ul.pagination {
     max-width: 100%;
     min-width: 100%;
 }
+
+#delta_urls_table_wrapper .col-md {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    grid-auto-flow: row;
+    position: relative;
+
+    .dt-info {
+        position:absolute;
+        left: 130px;
+        top: 5px;
+    }
+}
+
+#curated_urls_table_wrapper .col-md {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    grid-auto-flow: row;
+    position: relative;
+
+    .dt-info {
+        position:absolute;
+        left: 130px;
+        top: 5px;
+    }
+}

sde_indexing_helper/static/js/delta_url_list.js

@@ -114,14 +114,15 @@ function initializeDataTable() {
     layout: {
       bottomEnd: "inputPaging",
       topEnd: null,
-      topStart: {
-        info: true,
+      topStart: null,
+      top: {
         pageLength: {
           menu: [
             [25, 50, 100, 500],
             ["Show 25", "Show 50", "Show 100", "Show 500"],
           ],
         },
+        info:true,
         buttons: [
           {
             extend: "csv",
@@ -332,14 +333,15 @@ function initializeDataTable() {
     layout: {
       bottomEnd: "inputPaging",
       topEnd: null,
-      topStart: {
-        info: true,
+      topStart: null,
+      top: {
         pageLength: {
           menu: [
             [25, 50, 100, 500],
             ["Show 25", "Show 50", "Show 100", "Show 500"],
           ],
         },
+        info:true,
         buttons: [
           {
             extend: "csv",
@@ -1208,12 +1210,22 @@ function getCuratedScrapedTitleColumn() {
   };
 }
 
+function escapeHtml(str) {
+  if (!str) return '';
+  return str
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#39;');
+}
+
 function getGeneratedTitleColumn() {
   return {
     data: "generated_title",
     width: "20%",
     render: function (data, type, row) {
-      return `<input type="text" class="form-control individual_title_input whiteText" value='${data}' data-generated-title-id=${row["generated_title_id"]
+      return `<input type="text" class="form-control individual_title_input whiteText" value="${escapeHtml(data)}" data-generated-title-id=${row["generated_title_id"]
         } data-match-pattern-type=${row["match_pattern_type"]
         } data-delta-urls-count=${row["delta_urls_count"]
         } data-url=${remove_protocol(row["url"])} />`;
@@ -1226,7 +1238,7 @@ function getCuratedGeneratedTitleColumn() {
     data: "generated_title",
     width: "20%",
     render: function (data, type, row) {
-      return `<input type="text" class="form-control individual_title_input whiteText" value='${data}' data-generated-title-id=${row["generated_title_id"]
+      return `<input type="text" class="form-control individual_title_input whiteText" value="${escapeHtml(data)}" data-generated-title-id=${row["generated_title_id"]
         } data-match-pattern-type=${row["match_pattern_type"]
         } data-curated-urls-count=${row["curated_urls_count"]
         } data-url=${remove_protocol(row["url"])} />`;
@@ -2065,6 +2077,12 @@ $("#document_type_pattern_form").on("submit", function (e) {
     inputs[field.name] = field.value;
   });
 
+  // Validate that the document_type_pattern field is not empty
+  if (!inputs.document_type_pattern) {
+    toastr.error("Please select a Document Type");
+    return; // Prevent form submission
+  }
+
   postDocumentTypePatterns(
     inputs.match_pattern,
     inputs.match_pattern_type,

sde_indexing_helper/templates/includes/scripts.html

@@ -38,18 +38,14 @@
 <script src="{% static 'js/plugins/nouislider.min.js' %}"
         type="text/javascript"></script>
 <!-- Include a polyfill for ES6 Promises (optional) for IE11, UC Browser and Android browser support SweetAlert -->
-<script src="//cdnjs.cloudflare.com/ajax/libs/core-js/2.4.1/core.js"
-        type="text/javascript"></script>
-<script src="//cdnjs.cloudflare.com/ajax/libs/js-cookie/2.2.1/js.cookie.min.js"
-        type="text/javascript"></script>
-<script src="//unpkg.com/[email protected]"
-        integrity="sha384-KReoNuwj58fe4zgWyjj5a1HrvXYPBeV0a3bNPVjK7n5FdsGC41fHRx6sq5tONeP0"
-        crossorigin="anonymous"
-        type="text/javascript"></script>
-<script src="//cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/toastr.min.js"
-        integrity="sha512-VEd+nq25CkR676O+pLBnDW09R7VQX9Mdiij052gVCp5yVH3jGtH70Ho/UUv4mJDsEdTvqRCFZg0NKGiojGnUCw=="
-        crossorigin="anonymous"
-        referrerpolicy="no-referrer"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/core-js/2.4.1/core.js"
+        type="text/javascript" integrity="sha384-Ltf3zlo018jgSFarBV4ZXF8GxwymfafIj3qWz3rrjhL8hTVd2XzglHH+BCuIKnbk" crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/js-cookie/2.2.1/js.cookie.min.js"
+        type="text/javascript" integrity="sha384-eITc5AorI6xzkW7XunGaNrcA0l6qrU/kA/mOhLQOC5thAzlHSClQTOecyzGK6QXK" crossorigin="anonymous"></script>
+<script src="https://unpkg.com/[email protected]"
+        type="text/javascript" integrity="sha384-KReoNuwj58fe4zgWyjj5a1HrvXYPBeV0a3bNPVjK7n5FdsGC41fHRx6sq5tONeP0" crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/toastr.min.js"
+        type="text/javascript" integrity="sha384-Si3HKTyQYGU+NC4aAF3ThcOSvK+ZQiyEKlYyfjiIFKMqsnCmfHjGa1VK1kYP9UdS" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 <!-- Library for adding dinamically elements -->
 <script src="{% static 'js/plugins/arrive.min.js' %}" type="text/javascript"></script>
 <!-- Chartist JS -->

sde_indexing_helper/templates/sde_collections/collection_detail.html

@@ -8,15 +8,15 @@
 {% block stylesheets %}
 {{ block.super }}
 <link rel="stylesheet" href="{% static 'css/collection_detail.css' %}">
-<link href="//cdn.datatables.net/v/bs4/jszip-3.10.1/dt-2.0.5/af-2.7.0/b-3.0.2/b-colvis-3.0.2/b-html5-3.0.2/b-print-3.0.2/cr-2.0.1/fc-5.0.0/fh-4.0.1/kt-2.12.0/r-3.0.2/rg-1.5.0/rr-1.5.0/sc-2.4.1/sp-2.3.1/sl-2.0.1/datatables.min.css" rel="stylesheet">
+<link href="https://cdn.datatables.net/v/bs4/jszip-3.10.1/dt-2.0.5/af-2.7.0/b-3.0.2/b-colvis-3.0.2/b-html5-3.0.2/b-print-3.0.2/cr-2.0.1/fc-5.0.0/fh-4.0.1/kt-2.12.0/r-3.0.2/rg-1.5.0/rr-1.5.0/sc-2.4.1/sp-2.3.1/sl-2.0.1/datatables.min.css" rel="stylesheet">
 {% endblock stylesheets %}
 {% block javascripts %}
     {{ block.super }}
-    <script src="//cdnjs.cloudflare.com/ajax/libs/pdfmake/0.2.7/pdfmake.min.js"></script>
-    <script src="//cdnjs.cloudflare.com/ajax/libs/pdfmake/0.2.7/vfs_fonts.js"></script>
-    <script src="//cdn.datatables.net/v/bs4/jszip-3.10.1/dt-2.0.5/af-2.7.0/b-3.0.2/b-colvis-3.0.2/b-html5-3.0.2/b-print-3.0.2/cr-2.0.1/fc-5.0.0/fh-4.0.1/kt-2.12.0/r-3.0.2/rg-1.5.0/rr-1.5.0/sc-2.4.1/sp-2.3.1/sl-2.0.1/datatables.min.js"></script>
-    <script src="//cdnjs.cloudflare.com/ajax/libs/jquery.blockUI/2.70/jquery.blockUI.min.js"></script>
-    <script src="https://cdn.datatables.net/plug-ins/2.0.8/features/inputPaging/dist/dataTables.inputPaging.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.2.7/pdfmake.min.js" integrity="sha384-VFQrHzqBh5qiJIU0uGU5CIW3+OWpdGGJM9LBnGbuIH2mkICcFZ7lPd/AAtI7SNf7" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.2.7/vfs_fonts.js" integrity="sha384-/RlQG9uf0M2vcTw3CX7fbqgbj/h8wKxw7C3zu9/GxcBPRKOEcESxaxufwRXqzq6n" crossorigin="anonymous"></script>
+    <script src="https://cdn.datatables.net/v/bs4/jszip-3.10.1/dt-2.0.5/af-2.7.0/b-3.0.2/b-colvis-3.0.2/b-html5-3.0.2/b-print-3.0.2/cr-2.0.1/fc-5.0.0/fh-4.0.1/kt-2.12.0/r-3.0.2/rg-1.5.0/rr-1.5.0/sc-2.4.1/sp-2.3.1/sl-2.0.1/datatables.min.js" integrity="sha384-hYG8UYaiy9eSRy58/jaXhT3suoM1QfgYThNxzYEE1y+ijcFXuwMaNt1zr1ERbrIM" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.blockUI/2.70/jquery.blockUI.min.js" integrity="sha384-Ct2s0NBxEbvJlnXHOZJheqOGKjX3Q4ewsYoJZYnLz/teMXnlGhim5o9305EkvlsN" crossorigin="anonymous"></script>
+    <script src="https://cdn.datatables.net/plug-ins/2.0.8/features/inputPaging/dist/dataTables.inputPaging.min.js" integrity="sha384-mKZ8fubNsDhTYk1d7pc3PkY4PtNykywmSWAs9uCX02tybo9mFN0D842g4XkshSF/" crossorigin="anonymous"></script>
     <script src="{% static 'js/collection_detail.js' %}"></script>
     <script src="{% static 'js/core/bootstrap.min.js' %}"></script>
 {% endblock javascripts %}

sde_indexing_helper/templates/sde_collections/collection_list.html

@@ -5,7 +5,7 @@
 {% block stylesheets %}
 {% load humanize %}
     {{ block.super }}
-    <link href="//cdn.datatables.net/v/bs4/jszip-3.10.1/dt-2.0.5/af-2.7.0/b-3.0.2/b-colvis-3.0.2/b-html5-3.0.2/b-print-3.0.2/cr-2.0.1/fc-5.0.0/fh-4.0.1/kt-2.12.0/r-3.0.2/rg-1.5.0/rr-1.5.0/sc-2.4.1/sp-2.3.1/sl-2.0.1/datatables.min.css" rel="stylesheet">
+    <link href="https://cdn.datatables.net/v/bs4/jszip-3.10.1/dt-2.0.5/af-2.7.0/b-3.0.2/b-colvis-3.0.2/b-html5-3.0.2/b-print-3.0.2/cr-2.0.1/fc-5.0.0/fh-4.0.1/kt-2.12.0/r-3.0.2/rg-1.5.0/rr-1.5.0/sc-2.4.1/sp-2.3.1/sl-2.0.1/datatables.min.css" rel="stylesheet">
     <link rel="stylesheet" href="{% static 'css/collections_list.css' %}">
 {% endblock stylesheets %}
 {% block content %}
@@ -278,9 +278,9 @@ <h5 class="modalTitle whiteText" id="hideShowColumnsModalTitle">Customize Column
     </div>
 {% endblock content %}
 {% block javascripts %}
-    <script src="//cdnjs.cloudflare.com/ajax/libs/pdfmake/0.2.7/pdfmake.min.js"></script>
-    <script src="//cdnjs.cloudflare.com/ajax/libs/pdfmake/0.2.7/vfs_fonts.js"></script>
-    <script src="//cdn.datatables.net/v/bs4/jszip-3.10.1/dt-2.0.5/af-2.7.0/b-3.0.2/b-colvis-3.0.2/b-html5-3.0.2/b-print-3.0.2/cr-2.0.1/fc-5.0.0/fh-4.0.1/kt-2.12.0/r-3.0.2/rg-1.5.0/rr-1.5.0/sc-2.4.1/sp-2.3.1/sl-2.0.1/datatables.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.2.7/pdfmake.min.js" integrity="sha384-VFQrHzqBh5qiJIU0uGU5CIW3+OWpdGGJM9LBnGbuIH2mkICcFZ7lPd/AAtI7SNf7" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.2.7/vfs_fonts.js" integrity="sha384-/RlQG9uf0M2vcTw3CX7fbqgbj/h8wKxw7C3zu9/GxcBPRKOEcESxaxufwRXqzq6n" crossorigin="anonymous"></script>
+    <script src="https://cdn.datatables.net/v/bs4/jszip-3.10.1/dt-2.0.5/af-2.7.0/b-3.0.2/b-colvis-3.0.2/b-html5-3.0.2/b-print-3.0.2/cr-2.0.1/fc-5.0.0/fh-4.0.1/kt-2.12.0/r-3.0.2/rg-1.5.0/rr-1.5.0/sc-2.4.1/sp-2.3.1/sl-2.0.1/datatables.min.js" integrity="sha384-hYG8UYaiy9eSRy58/jaXhT3suoM1QfgYThNxzYEE1y+ijcFXuwMaNt1zr1ERbrIM" crossorigin="anonymous"></script>
     <script src="{% static 'js/collection_list.js' %}"></script>
     <script src="{% static 'js/project.js' %}"></script>
 {% endblock javascripts %}