feat(elasticsearch): Optimize Elasticsearch memory settings and environment config

.env.tpl

@@ -1,7 +1,5 @@
 # Elasticsearch Configuration
 ENV=${ENV}
-
-STACK_VERSION=${STACK_VERSION}
 CLUSTER_NAME=${CLUSTER_NAME}
 MEM_LIMIT=${MEM_LIMIT}
 SEED_HOSTS=${SEED_HOSTS}
@@ -13,6 +11,7 @@ ES_HOSTNAME=${ES_HOSTNAME}
 ES_USERNAME=${ES_USERNAME}
 ES_PASSWORD=${ES_PASSWORD}
 ES_CERTIF=${ES_CERTIF}
+ES_STACK_VERSION=${ES_STACK_VERSION}
 
 # Host System Configuration
 HOST_UID=${HOST_UID}

.github/workflows/main.yaml

@@ -9,14 +9,14 @@ on:
 env:
   ENV: dev
   # Elasticsearch Configuration
-  STACK_VERSION: 8.17.2
+  ES_STACK_VERSION: 8.17.2
   CLUSTER_NAME: docker-cluster
   MEM_LIMIT: 4g
   LICENSE: basic
   SSL_VERIFY_MODE: certificate
   #
   ES_PORT: 9200
-  ES_HOSTNAME: http://staging.literev.com:9200
+  ES_HOSTNAME: http://127.0.0.1:9200
   ES_USERNAME: elastic
   ES_PASSWORD: worksfine
   ES_CERTIF: containers/esconfig/certs/http_ca.crt
@@ -26,64 +26,78 @@ env:
   ES_HOST_VOLUME: /opt/services/data/elasticsearch/data
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     concurrency:
-      group: ci-${{ github.ref }}
+      group: ci-${{ github.event_name }}-${{ github.ref }}
       cancel-in-progress: true
 
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
-    - uses: actions/checkout@v3
-
-    - uses: mamba-org/setup-micromamba@v1
-      with:
-        miniconda-version: "latest"
-        mamba-version: "*"
-        environment-file: conda/base.yaml
-        channels: conda-forge,nodefaults
-        channel-priority: true
-        activate-environment: es-journals
-        use-mamba: true
-        miniforge-variant: Mambaforge
-
-    - name: Set Environment Variable
-      run: echo "GITHUB_PAT=${{ secrets.DEVTOOLS_PAT }}" >> $GITHUB_ENV
-
-    - name: Create environment variables files
-      run: |
-        envsubst < .env.tpl > .env
-
-    - name: Check if poetry.lock is updated
-      run: poetry check
-
-    - name: Install dependencies
-      run: poetry install --no-root
-
-    - name: Clean-up
-      run: makim clean.tmp
-
-    - name: Setup development environment for Rxivs
-      run: |
-        for rxiv_name in biorxiv medrxiv; do
-            makim develop.setup-dev-env --index-name $rxiv_name
-        done
-
-    - name: Start Elasticsearch
-      run: |
-        sugar compose-ext start --options "-d"
-
-    - name: Fetch and process MedRxiv data
-      run: |
-        bash scripts/fetch_rxivx_data.sh medrxiv
-
-    - name: Count documents in MedRxiv index
-      run: |
-        makim develop.count-docs-in-index --index-name medrxiv
-
-    - name: Run tests
-      run: makim tests.unittest
+      - uses: actions/checkout@v4
+
+      - name: Cache Conda Packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.conda/envs/es-journals
+          key: ubuntu-latest-conda-${{ hashFiles('conda/base.yaml') }}
+          restore-keys: |
+            ubuntu-latest-conda-
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          environment-file: conda/base.yaml
+          channels: conda-forge,nodefaults
+          activate-environment: es-journals
+          auto-update-conda: true
+          conda-solver: libmamba
+
+      - name: Set Environment Variable
+        run: echo "GITHUB_PAT=${{ secrets.DEVTOOLS_PAT }}" >> $GITHUB_ENV
+
+      - name: Create environment variables files
+        run: |
+          envsubst < .env.tpl > .env
+
+      - name: Check if poetry.lock is updated
+        run: poetry check
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.create false
+          poetry install --no-root
+
+      - name: Clean-up
+        run: makim clean.tmp
+
+      - name: Setup development environment for Rxivs
+        run: |
+          for rxiv_name in biorxiv medrxiv; do
+              makim develop.setup-dev-env --index-name $rxiv_name
+          done
+
+      - name: Start Elasticsearch
+        run: |
+          sugar compose-ext start --options "-d"
+
+      - name: Fetch and process MedRxiv data
+        run: |
+          bash scripts/fetch_rxivx_data.sh medrxiv
+
+      # - name: Count documents in MedRxiv index
+      #   run: |
+      #     makim develop.count-docs-in-index --index-name medrxiv
+
+
+      - name: Run tests
+        run: makim tests.unittest
+
+      - name: Setup tmate session
+        if: "${{ failure() }}"
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 360

containers/compose.elasticsearch.dev.yaml

@@ -1,14 +1,14 @@
 services:
   es:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ES_STACK_VERSION}
     container_name: es
-    user: "1000:1000"
+    user: "${HOST_UID:-1000}:${HOST_UID:-1000}"
     env_file:
       ../.env
     environment:
       - node.name=es
       - discovery.type=single-node
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "ES_JAVA_OPTS=-Xmx2g -Xmx2g"
       - bootstrap.memory_lock=false
       - ELASTIC_PASSWORD=${ES_PASSWORD}
     ulimits:

containers/compose.elasticsearch.staging.yaml

@@ -1,13 +1,14 @@
 services:
   es:
     container_name: es
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
-    user: "1000:1000" # Set the UID:GID to run the container
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ES_STACK_VERSION}
+    user: "${HOST_UID:-1000}:${HOST_UID:-1000}"
     env_file:
       - ../.env
     environment:
       - discovery.type=single-node
       - node.name=es
+      - "ES_JAVA_OPTS=-Xmx2g -Xmx2g"
       - ELASTIC_PASSWORD=${ES_PASSWORD}
       - ES_JAVA_OPTS=-Xms1g -Xmx1g  # Ensure proper memory allocation
       - bootstrap.memory_lock=true

containers/compose.elasticsearch.yaml

@@ -1,6 +1,7 @@
 services:
   es01:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ES_STACK_VERSION}
+    user: "${HOST_UID:-1000}:${HOST_UID:-1000}"
     volumes:
       - type: bind
         source: ${NGINX_CONFIG}/certificates
@@ -13,13 +14,17 @@ services:
         source: ${ES_HOST_VOLUME}
         target: /usr/share/elasticsearch/data
     ports:
-      - "${ES_PORT}:9200"
+      - "9200:9200"
     env_file: ../.env
     environment:
       - node.name=es01
       - cluster.name=${CLUSTER_NAME}
       - cluster.initial_master_nodes=${SEED_HOSTS}
       - discovery.seed_hosts=${SEED_HOSTS}
+      - cluster.initial_master_nodes=es01
+      - discovery.seed_hosts=es01
+      - discovery.type=single-node
+      - "ES_JAVA_OPTS=-Xmx2g -Xmx2g"
       - ES_PASSWORD=${ES_PASSWORD}
       - bootstrap.memory_lock=true
       - "xpack.security.enabled=true"

scripts/index_arxiv_to_es.py

@@ -22,7 +22,7 @@
     f"/tmp/elasticrxivx_{index_name_placeholder}_{time.strftime('%Y%m%d-%H%M%S')}.log"
 )
 
-ES_HOSTNAME = os.getenv("ES_HOSTNAME", "https://localhost:9200")
+ES_HOSTNAME = os.getenv("ES_HOSTNAME", "http://127.0.0.1:9200")
 ES_USERNAME = os.getenv("ES_USERNAME", "elastic")
 ES_PASSWORD = os.getenv("ES_PASSWORD", "")
 ES_CERTIF = os.getenv("ES_CERTIF", "")