Update libraries, NGINX and scripts

.gitignore

@@ -99,7 +99,7 @@ ENV/
 .idea/
 
 # Containers
-containers/nginx/*
+containers/nginx/data/*
 
 # Databases
 **data/*

.makim.yaml

@@ -194,3 +194,21 @@ groups:
           crontab -u devops $CRON_PATH
 
           echo "Cron jobs for LiteRev Elasticsearch have been set up successfully for user develop."
+
+  nginx:
+    tasks:
+      proxydomain:
+        help: Create SSL proxy configuration with CertBot for a domain
+        env-file: .env
+        run: |
+          python ./containers/nginx/scripts/generate_proxy_ui.py
+      generate-certificate:
+        help: Generate certificate
+        run: |
+          ./containers/nginx/scripts/generate-certificates.sh
+      setup:
+        hooks:
+          pre-run:
+            - task: clean.tmp
+            - task: nginx.proxydomain
+            - task: nginx.generate-certificate

.sugar.yaml

@@ -1,16 +1,20 @@
 backend: compose
 env-file: .env
 defaults:
-  group: ${{ env.SUGAR_ENV or env.ENV }}
+  profile: ${{ env.SUGAR_ENV or env.ENV }}
   project-name: literev-es
-groups:
+
+profiles:
   prod:
     config-path:
       - containers/compose.nginx.yaml
       - containers/compose.elasticsearch.yaml
     env-file: .env
     services:
-      default: es01,nginx,certbot
+      default:
+        - es01
+        - nginx
+        - certbot
       available:
         - name: es01
         - name: nginx
@@ -20,14 +24,16 @@ groups:
       - containers/compose.elasticsearch.dev.yaml
     env-file: .env
     services:
-      default: es
+      default:
+        - es
       available:
         - name: es
   staging:
     config-path:
       - containers/compose.elasticsearch.staging.yaml
     env-file: .env
     services:
-      default: es
+      default:
+        - es
       available:
         - name: es

conda/base.yaml

@@ -1,21 +1,23 @@
+# conda/base.yaml
 name: es-journals
 channels:
-  - nodefaults
   - conda-forge
+  - nodefaults
 dependencies:
+  - python >=3.10,<3.13 # <-- was <13.3; pin to avoid 3.14
   - elasticsearch
   - loguru
-  - nodejs  # used by semantic-release
-  - python >=3.8.1,<13.3
+  - nodejs
   - pip
   - poetry
   - r-base
   - r-devtools
   - shellcheck
   - typer
+  - python-levenshtein # prebuilt from conda-forge; avoids pip build
+  - pytest
+  - tqdm
   - pip:
     - compose-go
-    - makim==1.20.0
-    - containers-sugar
-    - pytest
-    - tqdm
+    - makim>=1.27.0
+    - containers-sugar>=1.19.2

containers/compose.elasticsearch.yaml

@@ -2,45 +2,36 @@ services:
   es01:
     image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
     volumes:
-      - type: bind
-        source: ${NGINX_CONFIG}/certificates
-        target: /usr/share/elasticsearch/config/certs
       - type: bind
         source: ../containers/init-scripts
         target: /usr/share/elasticsearch/init-scripts
         read_only: true
+      - type: bind
+        source: ${HOST_ELASTIC_CERTS}
+        target: /usr/share/elasticsearch/config/letsencrypt/live/${CERTBOT_DOMAIN}/
+        read_only: true
       - type: bind
         source: ${ES_HOST_VOLUME}
         target: /usr/share/elasticsearch/data
-    ports:
-      - "${ES_PORT}:9200"
-    env_file: ../.env
     environment:
       - node.name=es01
-      - cluster.name=${CLUSTER_NAME}
-      - cluster.initial_master_nodes=${SEED_HOSTS}
-      - discovery.seed_hosts=${SEED_HOSTS}
-      - ES_PASSWORD=${ES_PASSWORD}
-      - bootstrap.memory_lock=true
+      - cluster.name=docker-cluster
+      - discovery.type=single-node
+
       - "xpack.security.enabled=true"
-      - "xpack.security.http.ssl.enabled=true"
-      - "xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certs/privkey.pem"
-      - "xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certs/fullchain.pem"
-      - "xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certs/chain.pem"
-      - "xpack.security.http.ssl.verification_mode=${SSL_VERIFY_MODE}"
-      - "xpack.security.transport.ssl.enabled=true"
-      - "xpack.security.transport.ssl.key=/usr/share/elasticsearch/config/certs/privkey.pem"
-      - "xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/config/certs/fullchain.pem"
-      - "xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/config/certs/chain.pem"
-      - "xpack.security.transport.ssl.verification_mode=${SSL_VERIFY_MODE}"
-      - "xpack.license.self_generated.type=${LICENSE}"
+      - "xpack.security.http.ssl.enabled=false"
+      - "xpack.security.transport.ssl.enabled=false"
+      - ELASTIC_PASSWORD=${ES_PASSWORD}
+    ports:
+      - "${ES_PORT}:9200"
+    env_file: ../.env
     mem_limit: ${MEM_LIMIT:-4g}
     ulimits:
       memlock:
         soft: -1
         hard: -1
     healthcheck:
-      test: ["CMD-SHELL", "curl -k -u elastic:$(cat /run/secrets/es_password) https://es01:9200/_cluster/health"]
+      test: ["CMD-SHELL", "curl -s -u elastic:${ES_PASSWORD} http://localhost:9200/_cluster/health || exit 1"]
       interval: 10s
       timeout: 10s
       retries: 120

containers/compose.nginx.yaml

@@ -1,29 +1,24 @@
 services:
   nginx:
-    build:
-      context: ".."
-      dockerfile: "containers/dockerfile.nginx" # Relative path
-      args:
-        HOST_UID: ${HOST_UID}
-        HOST_GID: ${HOST_GID}
+    image: nginx:1.25
     env_file:
       - ../.env
-    container_name: nginx
     ports:
       - "80:80"
       - "443:443"
     volumes:
-      - ${NGINX_CONFIG}:/etc/nginx # Relative path
-      - ${NGINX_CONFIG}/certbot/conf:/etc/nginx/letsencrypt
+      - ${NGNIX_CONF}:/etc/nginx/conf.d
+      - ${CERTBOT_CONF}:/etc/letsencrypt
+      - ${CERTBOT_WWW}:/var/www/certbot
     networks:
       - elastic
 
   certbot:
     image: certbot/certbot
     container_name: certbot
     volumes:
-      - ${NGINX_CONFIG}/certbot/conf:/etc/letsencrypt
-      - ${NGINX_CONFIG}/certbot/www:/var/www/certbot
+      - ${NGNIX_CONF}/certbot/conf:/etc/letsencrypt
+      - ${NGNIX_CONF}/certbot/www:/var/www/certbot
     entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
     networks:
       - elastic

containers/nginx/scripts/create_symlinks.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+SOURCE_DIR="/etc/nginx/sites-available"
+DEST_DIR="/etc/nginx/sites-enabled"
+
+for file in "$SOURCE_DIR"/*; do
+    filename=$(basename "$file")
+    ln -s "$file" "$DEST_DIR/$filename"
+done

containers/nginx/scripts/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+certbot certonly --nginx -n --agree-tos -d ${CERTBOT_DOMAIN} -d www.${CERTBOT_DOMAIN} -m ${CERTBOT_EMAIL}
+
+# Update the Nginx configuration to use the newly generated SSL certificates
+sed -i "s/\/tmp\/nginx-selfsigned.crt/\/etc\/letsencrypt\/live\/${CERTBOT_DOMAIN}\/fullchain.pem/" /etc/nginx/sites-available/*
+sed -i "s/\/tmp\/nginx-selfsigned.key/\/etc\/letsencrypt\/live\/${CERTBOT_DOMAIN}\/privkey.pem/" /etc/nginx/sites-available/*

containers/nginx/scripts/generate-certificates.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+set -ex
+
+# Check if sugar is avaiable
+if ! [ -x "$(command -v sugar)" ]; then
+    echo "Error: sugar is not installed." >&2
+    exit 1
+fi
+
+# Paths
+script_dir="$(cd "$(dirname "${BASH_SOURCE[1]}")" && pwd)"
+project_dir="$script_dir/"
+
+# Check .env exists for environment variabes
+if [[ -f "$project_dir/.env" ]]; then
+  echo "Loading .env file..."
+  source "$project_dir/.env"
+else
+  echo 'Error: .env file not found.' >&2
+  exit 1
+fi
+
+## Check if ENV is set to prod or staging
+if [[ "${ENV:-}" != "prod" && "${ENV:-}" != "staging" ]]; then
+  echo 'Error: ENV should be set to prod or staging.' >&2
+  exit 1
+fi
+
+# Check if CERTBOT_DOMAIN is set and not empty
+if [[ -z "${CERTBOT_DOMAIN:-}" ]]; then
+  echo 'Error: CERTBOT_DOMAIN is not set or empty.' >&2
+  exit 1
+fi
+
+# Check if CERTBOT_EMAIL is set and not empty
+if [[ -z "${CERTBOT_EMAIL:-}" ]]; then
+  echo 'Error: CERTBOT_EMAIL is not set or empty.' >&2
+  exit 1
+fi
+
+DOMAINS=("$CERTBOT_DOMAIN")
+RSA_KEY_SIZE=4096
+EMAIL="$CERTBOT_EMAIL"
+STAGING="${STAGING:-0}"  # use STAGING=1 in env to hit Let's Encrypt staging
+
+echo "----> Creating dummy certificate for ${DOMAINS[*]} ..."
+path="/etc/letsencrypt/live/$CERTBOT_DOMAIN"
+
+# mkdir -p target path inside certbot container
+sugar compose run --service certbot --cmd "/bin/sh -lc 'mkdir -p \"$path\"'" -- --rm
+
+# Create a 1-day dummy cert so nginx can boot with TLS paths present
+sugar compose run --service certbot --cmd "/bin/sh -lc '\
+  openssl req -x509 -nodes -newkey rsa:$RSA_KEY_SIZE -days 1 \
+    -keyout \"$path/privkey.pem\" \
+    -out \"$path/fullchain.pem\" \
+    -subj \"/CN=localhost\"'" -- --rm
+
+echo "----> Starting nginx ..."
+sugar compose up --services literev-nginx -- --force-recreate -d
+
+echo "----> Deleting dummy certificate for ${DOMAINS[*]} ..."
+sugar compose run --service certbot --entrypoint "\
+  rm -rf /etc/letsencrypt/live/$CERTBOT_DOMAIN \
+         /etc/letsencrypt/archive/$CERTBOT_DOMAIN \
+         /etc/letsencrypt/renewal/$CERTBOT_DOMAIN.conf" -- --rm
+
+echo "----> Requesting Let's Encrypt certificate for $DOMAINS ..."
+domain_args=""
+for domain in "${DOMAINS[@]}"; do
+    domain_args="$domain_args -d $domain"
+done
+
+# Select appropriate email arg
+case "$EMAIL" in
+    "") email_arg="--register-unsafely-without-email" ;;
+    *) email_arg="--email $EMAIL" ;;
+esac
+
+# Enable staging mode if needed
+staging_arg=""
+if [ "${STAGING:-0}" != "0" ]; then
+    staging_arg="--dry-run"
+fi
+
+echo "----> certbot certonly..."
+# Run certbot to get real certificates
+sugar compose run --service certbot --cmd "/bin/sh -lc '\
+  certbot certonly --webroot -w /var/www/certbot \
+    ${staging_arg:-} \
+    ${email_arg:-} \
+    ${domain_args} \
+    --rsa-key-size $RSA_KEY_SIZE \
+    --agree-tos \
+    --no-eff-email \
+    --force-renewal'" -- --rm
+
+echo "----> Reloading nginx ..."
+sugar compose exec --service literev-nginx --cmd "nginx -s reload"

containers/nginx/scripts/generate_proxy_ui.py

@@ -0,0 +1,44 @@
+"""Generate proxy configuration."""
+
+from __future__ import annotations
+
+import os
+
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+
+ENV = os.environ.get("ENV", "staging")
+DOMAIN_ENV = os.environ.get("DOMAIN_ENV")
+
+print(f"Generating nginx config for: {DOMAIN_ENV}...")
+
+# Set up Jinja2 environment
+templates = Environment(
+    loader=FileSystemLoader(Path(__file__).parent.parent / "templates"),
+    autoescape=True,
+)
+
+nginx_template = templates.get_template("nginx.template.conf")
+domains = os.environ.get("CERTBOT_DOMAIN", "")
+
+# Define variables for the template
+variables = {
+    "upstream_name": "webapp",
+    "upstream_service": "literev",  # django container hostname
+    "upstream_port": os.environ.get("FRONTEND_HOST_PORT", 8000),
+    "certbot_root": "/var/www/certbot",
+    "letsencrypt_root": "/etc/letsencrypt",
+    "domain": domains.split(",")[0],  # Prevents www.
+}
+
+# Render the template with variables
+output = nginx_template.render(variables)
+
+nginx_dir = Path(__file__).parent.parent / "data" / "config" / f"{ENV}"
+
+nginx_dir.mkdir(parents=True, exist_ok=True)
+
+output_file = nginx_dir / f"{DOMAIN_ENV}.nginx.conf"
+
+output_file.write_text(output)