Merge pull request #974 from NASA-IMPACT/2429-implement-the-frontend-and-backend-for-url-based-division-picker

Division by URL feature

Author: code-geek

README.md

@@ -84,7 +84,7 @@ Navigate to the server running prod, then to the project folder. Run the followi
 ```bash
 docker-compose -f production.yml run --rm --user root django python manage.py dumpdata --natural-foreign --natural-primary --exclude=contenttypes --exclude=auth.Permission --indent 2 --output /app/backups/prod_backup-20240812.json
 ```
-This will have saved the backup in a folder outside of the docker container. Now you can copy it to your local machine. 
+This will have saved the backup in a folder outside of the docker container. Now you can copy it to your local machine.
 
 ```bash
 mv ~/prod_backup-20240812.json <project_path>/prod_backup-20240812.json

config/settings/local.py

@@ -26,9 +26,7 @@
 # EMAIL
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
-EMAIL_BACKEND = env(
-    "DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend"
-)
+EMAIL_BACKEND = env("DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend")
 
 # django-debug-toolbar
 # ------------------------------------------------------------------------------

config/settings/production.py

@@ -13,9 +13,7 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
 SECRET_KEY = env("DJANGO_SECRET_KEY")
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
-ALLOWED_HOSTS = env.list(
-    "DJANGO_ALLOWED_HOSTS", default=["sde-indexing-helper.nasa-impact.net"]
-)
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["sde-indexing-helper.nasa-impact.net"])
 
 # DATABASES
 # ------------------------------------------------------------------------------
@@ -36,15 +34,11 @@
 # TODO: set this to 60 seconds first and then to 518400 once you prove the former works
 SECURE_HSTS_SECONDS = 60
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-include-subdomains
-SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool(
-    "DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True
-)
+SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool("DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True)
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-preload
 SECURE_HSTS_PRELOAD = env.bool("DJANGO_SECURE_HSTS_PRELOAD", default=True)
 # https://docs.djangoproject.com/en/dev/ref/middleware/#x-content-type-options-nosniff
-SECURE_CONTENT_TYPE_NOSNIFF = env.bool(
-    "DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True
-)
+SECURE_CONTENT_TYPE_NOSNIFF = env.bool("DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True)
 
 # STORAGES
 # ------------------------------------------------------------------------------
@@ -61,9 +55,7 @@
 # DO NOT change these unless you know what you're doing.
 _AWS_EXPIRY = 60 * 60 * 24 * 7
 # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings
-AWS_S3_OBJECT_PARAMETERS = {
-    "CacheControl": f"max-age={_AWS_EXPIRY}, s-maxage={_AWS_EXPIRY}, must-revalidate"
-}
+AWS_S3_OBJECT_PARAMETERS = {"CacheControl": f"max-age={_AWS_EXPIRY}, s-maxage={_AWS_EXPIRY}, must-revalidate"}
 # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings
 AWS_S3_MAX_MEMORY_SIZE = env.int(
     "DJANGO_AWS_S3_MAX_MEMORY_SIZE",
@@ -128,12 +120,7 @@
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": True,
-    "formatters": {
-        "verbose": {
-            "format": "%(levelname)s %(asctime)s %(module)s "
-            "%(process)d %(thread)d %(message)s"
-        }
-    },
+    "formatters": {"verbose": {"format": "%(levelname)s %(asctime)s %(module)s " "%(process)d %(thread)d %(message)s"}},
     "handlers": {
         "console": {
             "level": "DEBUG",

config/urls.py

@@ -4,9 +4,7 @@
 from django.urls import include, path
 from django.views import defaults as default_views
 
-admin.site.site_header = (
-    "SDE Indexing Helper Administration"  # default: "Django Administration"
-)
+admin.site.site_header = "SDE Indexing Helper Administration"  # default: "Django Administration"
 admin.site.index_title = "SDE Indexing Helper"  # default: "Site administration"
 admin.site.site_title = "SDE Indexing Helper"  # default: "Django site admin"
 

config_generation/api.py

@@ -20,9 +20,7 @@
 
 class Api:
     def __init__(self, server_name: str) -> None:
-        self.headers: dict[str, str] = {
-            "Authorization": f"Bearer {tokens[server_name]}"
-        }
+        self.headers: dict[str, str] = {"Authorization": f"Bearer {tokens[server_name]}"}
         self.app_name: str = server_configs[server_name]["app_name"]
         self.query_name: str = server_configs[server_name]["query_name"]
         self.base_url: str = server_configs[server_name]["base_url"]
@@ -53,15 +51,11 @@ def query(self, term: str):
 
         return self.process_response(url, payload)
 
-    def sql(
-        self, source: str, collection: str = "", fetch_all: bool = False
-    ) -> dict[str, Any]:
+    def sql(self, source: str, collection: str = "", fetch_all: bool = False) -> dict[str, Any]:
         url = f"{self.base_url}/api/v1/engine.sql"
 
         collection_name = f"/{source}/{collection}/"
-        sql_command_all = (
-            "select url1,title,collection from @@ScienceMissionDirectorate"
-        )
+        sql_command_all = "select url1,title,collection from @@ScienceMissionDirectorate"
         if fetch_all:
             sql_command = sql_command_all
         else:

config_generation/config_example.py

@@ -1,6 +1,4 @@
-from sources_to_scrape import (
-    sources_to_index_test_grid_20240809,
-)
+from sources_to_scrape import sources_to_index_test_grid_20240809
 
 tokens: dict[str, str] = {
     "test_server": "token here",

config_generation/delete_server_content.py

@@ -2,7 +2,13 @@
 
 from db_to_xml_file_based import XmlEditor
 
-from config import batch_delete_name, collection_list, indexes_to_delete_from, source, engines
+from config import (
+    batch_delete_name,
+    collection_list,
+    engines,
+    indexes_to_delete_from,
+    source,
+)
 
 COMMAND_FILES_PATH = "../sinequa_configs/commands/"
 DELETE_COMMAND_TEMPLATE_PATH = "xmls/delete_template.xml"

config_generation/export_collections.py

@@ -44,17 +44,13 @@
     # Create JSON file
     print("Creating JSON dump...")
     json_data = json.dumps(bulk_data)
-    file_path = (
-        f"{TEMP_FOLDER_NAME}/{collection}/urls.json"  # Provide the desired file path
-    )
+    file_path = f"{TEMP_FOLDER_NAME}/{collection}/urls.json"  # Provide the desired file path
     with open(file_path, "w") as file:
         file.write(json_data)
 
     # Zip the JSON file
     print("Creating zip file...")
-    zip_file_path = (
-        f"{TEMP_FOLDER_NAME}/{collection}.zip"  # Provide the desired zip file path
-    )
+    zip_file_path = f"{TEMP_FOLDER_NAME}/{collection}.zip"  # Provide the desired zip file path
     with zipfile.ZipFile(zip_file_path, "w") as zip_file:
         zip_file.write(file_path, os.path.basename(file_path))
 

config_generation/export_whole_index.py

@@ -44,9 +44,7 @@
 
 # Upload the zip file to S3
 s3_bucket_name = env("DJANGO_AWS_STORAGE_BUCKET_NAME")
-s3_key = (
-    "scraped_urls_all/all_data.zip"  # Provide the desired S3 key for the uploaded file
-)
+s3_key = "scraped_urls_all/all_data.zip"  # Provide the desired S3 key for the uploaded file
 s3_client = boto3.client(
     "s3",
     region_name="us-east-1",

config_generation/generate_collection_list.py

@@ -22,9 +22,7 @@ def create_xml_path(collection_name):
 
 def get_turned_on_sources():
     # remove sources that were just scraped
-    turned_on_remaining_sources = [
-        source for source in turned_on_sources if source not in already_scraped_sources
-    ]
+    turned_on_remaining_sources = [source for source in turned_on_sources if source not in already_scraped_sources]
 
     # filter all sources to only webcrawler sources
     turned_on_remaining_webcrawlers = []
@@ -68,8 +66,5 @@ def get_sources_20230605():
     folders = get_all_config_folder_names()
     folders = folders + interrupted_sources
     return [
-        folder
-        for folder in folders
-        if folder not in sources_with_documents_20230605
-        and is_collection_crawler(folder)
+        folder for folder in folders if folder not in sources_with_documents_20230605 and is_collection_crawler(folder)
     ]