Merge branch 'dev' into dependabot/docker/compose/local/docs/python-3.10.14-slim-bullseye

Author: code-geek

.envs/.local/.django

@@ -33,3 +33,7 @@ SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH='dummy_branch'
 # Slack Webhook
 # ------------------------------------------------------------------------------
 SLACK_WEBHOOK_URL=''
+LRM_USER=''
+LRM_PASSWORD=''
+XLI_USER=''
+XLI_PASSWORD=''

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a bug report to help the development team fix any issues on COSMOS
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+## Describe the bug
+A clear and concise description of what the bug is.
+
+## Steps To Reproduce
+Steps to reproduce the behavior on https://sde-indexing-helper.nasa-impact.net/:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+## Expected behavior
+A clear and concise description of what you expected to happen originally.
+
+## Screenshots
+If applicable, add screenshots to help explain your problem.
+
+## Desktop (please complete the following information):
+ - OS: [e.g. iOS]
+ - Browser: [e.g. chrome, safari]
+ - Version: [e.g. 22]
+
+## Smartphone (please complete the following information):
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser: [e.g. stock browser, safari]
+ - Version: [e.g. 22]
+
+## Additional context
+Add any other context about the problem here.

README.md

@@ -78,7 +78,26 @@ To load collections:
 $ docker-compose -f local.yml run --rm django python manage.py loaddata sde_collections/fixtures/collections.json
 ```
 
-### Loading the Database from a Backup
+### Manually Creating and Loading a ContentTypeless Backup
+Navigate to the server running prod, then to the project folder. Run the following command to create a backup:
+
+```bash
+docker-compose -f production.yml run --rm --user root django python manage.py dumpdata --natural-foreign --natural-primary --exclude=contenttypes --exclude=auth.Permission --indent 2 --output /app/backups/prod_backup-20240812.json
+```
+This will have saved the backup in a folder outside of the docker container. Now you can copy it to your local machine. 
+
+```bash
+mv ~/prod_backup-20240812.json <project_path>/prod_backup-20240812.json
+scp sde:/home/ec2-user/sde_indexing_helper/backups/prod_backup-20240812.json prod_backup-20240812.json
+```
+
+Finally, load the backup into your local database:
+
+```bash
+docker-compose -f local.yml run --rm django python manage.py loaddata prod_backup-20240812.json
+```
+
+### Loading the Database from an Arbitrary Backup
 
 1. Build the project and run the necessary containers (as documented above).
 2. Clear out content types using the Django shell:

compose/local/django/Dockerfile

@@ -1,8 +1,8 @@
 # define an alias for the specfic python version used in this file.
-FROM python:3.10.13-slim-bullseye as python
+FROM python:3.10.13-slim-bullseye AS python
 
 # Python build stage
-FROM python as python-build-stage
+FROM python AS python-build-stage
 
 ARG BUILD_ENVIRONMENT=local
 
@@ -25,7 +25,7 @@ RUN pip wheel --wheel-dir /usr/src/app/wheels  \
 
 
 # Python 'run' stage
-FROM python as python-run-stage
+FROM python AS python-run-stage
 
 ARG BUILD_ENVIRONMENT=local
 ARG APP_HOME=/app

compose/local/docs/Dockerfile

@@ -1,9 +1,8 @@
 # define an alias for the specfic python version used in this file.
 FROM python:3.10.14-slim-bullseye as python
 
-
 # Python build stage
-FROM python as python-build-stage
+FROM python AS python-build-stage
 
 ENV PYTHONDONTWRITEBYTECODE 1
 
@@ -26,7 +25,7 @@ RUN pip wheel --no-cache-dir --wheel-dir /usr/src/app/wheels  \
 
 
 # Python 'run' stage
-FROM python as python-run-stage
+FROM python AS python-run-stage
 
 ARG BUILD_ENVIRONMENT
 ENV PYTHONUNBUFFERED 1

compose/production/django/Dockerfile

@@ -1,10 +1,8 @@
-
-
 # define an alias for the specfic python version used in this file.
-FROM python:3.10.13-slim-bullseye as python
+FROM python:3.10.14-slim-bullseye AS python
 
 # Python build stage
-FROM python as python-build-stage
+FROM python AS python-build-stage
 
 ARG BUILD_ENVIRONMENT=production
 
@@ -27,7 +25,7 @@ RUN pip wheel --wheel-dir /usr/src/app/wheels  \
 
 
 # Python 'run' stage
-FROM python as python-run-stage
+FROM python AS python-run-stage
 
 ARG BUILD_ENVIRONMENT=production
 ARG APP_HOME=/app

compose/production/traefik/Dockerfile

@@ -1,4 +1,4 @@
-FROM traefik:2.11.0
+FROM traefik:3.1.2
 RUN mkdir -p /etc/traefik/acme \
   && touch /etc/traefik/acme/acme.json \
   && chmod 600 /etc/traefik/acme/acme.json

config/settings/base.py

@@ -74,10 +74,21 @@
     "crispy_bootstrap5",
     "crispy_forms",
     "django_celery_beat",
+    "django_filters",
     "rest_framework_datatables",
     "rest_framework",
 ]
 
+LOCAL_APPS = [
+    "environmental_justice",
+    "feedback",
+    "sde_collections",
+    "sde_indexing_helper.users",
+]
+
+# https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
+INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
+
 CORS_ALLOWED_ORIGINS = [
     "http://localhost:3000",
     "http://sde-lrm.nasa-impact.net",
@@ -91,15 +102,6 @@
     "http://localhost:4200",
 ]
 
-LOCAL_APPS = [
-    "environmental_justice",
-    "sde_indexing_helper.users",
-    "sde_collections",
-    "feedback",
-]
-# https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
-INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
-
 # MIGRATIONS
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#migration-modules
@@ -339,3 +341,7 @@
 SINEQUA_CONFIGS_REPO_DEV_BRANCH = env("SINEQUA_CONFIGS_REPO_DEV_BRANCH")
 SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH = env("SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH")
 SLACK_WEBHOOK_URL = env("SLACK_WEBHOOK_URL")
+XLI_USER = env("XLI_USER")
+XLI_PASSWORD = env("XLI_PASSWORD")
+LRM_USER = env("LRM_USER")
+LRM_PASSWORD = env("LRM_PASSWORD")

config_generation/config_example.py

@@ -1,60 +1,49 @@
-# rename this file to config.py and do not track it on git
-# example on how to get the token are in the README.md
-# go down to the bottom to the section called this is the stuff you want to change
+from sources_to_scrape import (
+    sources_to_index_test_grid_20240809,
+)
 
-from sources_to_scrape import sources_to_delete_20231108
-
-# API Config
 tokens: dict[str, str] = {
     "test_server": "token here",
     "ren_server": "token here",
 }
 
 AVAILABLE_INDEXERS_TEST = [
-    "NodeIndexer1/identity0",
-    "NodeWebapp1/identity0",
-    "NodeWebapp2/identity0",
-    "NodeWebapp3/identity0",
+    "IndexerServerA/identity0",
+    "IndexerServerB/identity0",
 ]
 
 AVAILABLE_INDEXERS_PROD = ["NodeINDEX1/identity0", "NodeINDEX2/identity0"]
 
 TEST_SERVER_INDEXES = [  # this is the test server list
-    "HELIO_Repository_1",
-    "SDE_Acronyms",
-    "SMD_ASTRO_Repository_1",
-    "SMD_ASTRO_Repository_2",
-    "SMD_EARTHSCIENCE_Repository_1",
-    "SMD_GENELAB_Repository_1",
-    "SMD_PLANETARY_Repository_1",
-    "SMD_PLANETARY_Repository_2",
+    # "sde_neural_test_index",
+    "sde_index"
 ]
 
 PROD_SERVER_INDEXES = [
     # "EDP_Audit_1",
-    "SMD_LSDA_Repository_1",
-    # "EDP_UserMetadata_1",
-    "SMD_NTRS_Repository_1",
-    "GCMD_Repository_1",
-    "SMD_PLANETARY_Repository_1",
-    # "GCMD_Repository_1_Metadata",
-    "SMD_PLANETARY_Repository_2",
-    "GCMD_Repository_2",
-    "STI_Repository_1",
-    # "GCMD_Repository_3_Metadata",
-    # "STI_Repository_1_Metadata",
-    "HELIO_Repository_1",
-    "STI_Repository_2",
+    # "SMD_LSDA_Repository_1",
+    # # "EDP_UserMetadata_1",
+    # "SMD_NTRS_Repository_1",
+    # "GCMD_Repository_1",
+    # "SMD_PLANETARY_Repository_1",
+    # # "GCMD_Repository_1_Metadata",
+    # "SMD_PLANETARY_Repository_2",
+    # "GCMD_Repository_2",
+    # "STI_Repository_1",
+    # # "GCMD_Repository_3_Metadata",
+    # # "STI_Repository_1_Metadata",
+    # "HELIO_Repository_1",
+    # "STI_Repository_2",
     "SDE_Index",
     # "STI_Repository_2_Metadata",
-    "SMD_ASTRO_Repository_1",
-    "STI_Repository_3",
-    "SMD_ASTRO_Repository_2",
-    "STI_Repository_4",
-    "SMD_EARTHSCIENCE_Repository_1",
-    # "SinequaDoc",
-    "SMD_GENELAB_Repository_1",
-    "Test",
+    # "SMD_ASTRO_Repository_1",
+    # "STI_Repository_3",
+    # "SMD_ASTRO_Repository_2",
+    # "STI_Repository_4",
+    # "SMD_EARTHSCIENCE_Repository_1",
+    # # "SinequaDoc",
+    # "SMD_GENELAB_Repository_1",
+    # "Test",
 ]
 
 
@@ -69,12 +58,14 @@
     },
 }
 
-# this is the stuff you want to change
-collection_list: list[str] = sources_to_delete_20231108  # python list
-source = "SMD"
-batch_name: str = "delete_everywhere_20231108_test"
+# Job Creation Config
+collection_list: list[str] = sources_to_index_test_grid_20240809  # python list
+date = "20240809"
+source = "SDE"
 server = "test"
 
 # auto assigned
+batch_delete_name: str = f"sources_to_delete_on_{server}_{date}"
+batch_index_name: str = f"sources_to_index_on_{server}_{date}"
 available_indexers = SERVER_INFO[server]["indexers"]
 indexes_to_delete_from = SERVER_INFO[server]["indexes"]

config_generation/delete_server_content.py

@@ -2,16 +2,18 @@
 
 from db_to_xml_file_based import XmlEditor
 
-from config import batch_name, collection_list, indexes_to_delete_from, source
+from config import batch_delete_name, collection_list, indexes_to_delete_from, source, engines
 
 COMMAND_FILES_PATH = "../sinequa_configs/commands/"
 DELETE_COMMAND_TEMPLATE_PATH = "xmls/delete_template.xml"
 
 command_file = XmlEditor(DELETE_COMMAND_TEMPLATE_PATH)
 
+command_file.update_or_add_element_value(element_name="Engines", element_value=",".join([engine for engine in engines]))
 for collection in collection_list:
     for index in indexes_to_delete_from:
         sql = f"delete from {index} where collection='/{source}/{collection}/'"
         command_file.update_or_add_element_value(element_name="SQL", element_value=sql, add_duplicate=True)
-file_name = f"{COMMAND_FILES_PATH}{batch_name}.xml"
+
+file_name = f"{COMMAND_FILES_PATH}{batch_delete_name}.xml"
 command_file._update_config_xml(file_name)