feat: Elasticsearch access via API Key

fixed local configuration

feat: Elasticsearch access via API Key: reverse testconfig

Author: leonardlin

common/doc_store/es_conn_pool.py

@@ -25,7 +25,6 @@
 
 @singleton
 class ElasticSearchConnectionPool:
-
     def __init__(self):
         if hasattr(settings, "ES"):
             self.ES_CONFIG = settings.ES
@@ -52,12 +51,23 @@ def __init__(self):
             raise Exception(msg)
 
     def _connect(self):
-        self.es_conn = Elasticsearch(
-            self.ES_CONFIG["hosts"].split(","),
-            basic_auth=(self.ES_CONFIG["username"], self.ES_CONFIG[
-                "password"]) if "username" in self.ES_CONFIG and "password" in self.ES_CONFIG else None,
-            verify_certs= self.ES_CONFIG.get("verify_certs", False),
-            timeout=600 )
+        if "api_key" in self.ES_CONFIG and self.ES_CONFIG["api_key"]:
+            logging.info('Elasticsearch connection using API_KEY')
+            self.es_conn = Elasticsearch(
+                self.ES_CONFIG["hosts"].split(","), 
+                api_key=self.ES_CONFIG["api_key"], 
+                verify_certs=self.ES_CONFIG.get("verify_certs", False), 
+                timeout=600
+            )
+        else:
+            logging.info('Elasticsearch connection using username/password')
+            self.es_conn = Elasticsearch(
+                self.ES_CONFIG["hosts"].split(","),
+                basic_auth=(self.ES_CONFIG["username"], self.ES_CONFIG["password"]) if "username" in self.ES_CONFIG and "password" in self.ES_CONFIG else None,
+                verify_certs=self.ES_CONFIG.get("verify_certs", False),
+                timeout=600,
+            )
+
         if self.es_conn:
             self.info = self.es_conn.info()
             return True

conf/service_conf.yaml

@@ -23,6 +23,7 @@ es:
   hosts: 'http://localhost:1200'
   username: 'elastic'
   password: 'infini_rag_flow'
+  api_key: ''
 os:
   hosts: 'http://localhost:1201'
   username: 'admin'

docker/.env

@@ -31,7 +31,7 @@ COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE}
 STACK_VERSION=8.11.3
 
 # The hostname where the Elasticsearch service is exposed
-ES_HOST=es01
+ES_HOST=http://es01:9200
 
 # The port used to expose the Elasticsearch service to the host machine,
 # allowing EXTERNAL access to the service running inside the Docker container.
@@ -154,7 +154,7 @@ ADMIN_SVR_HTTP_PORT=9381
 SVR_MCP_PORT=9382
 
 # The RAGFlow Docker image to download. v0.22+ doesn't include embedding models.
-RAGFLOW_IMAGE=infiniflow/ragflow:v0.24.0
+RAGFLOW_IMAGE=ragflow:v0.24.0
 
 # If you cannot download the RAGFlow Docker image:
 # RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:v0.24.0

docker/service_conf.yaml.template

@@ -20,9 +20,10 @@ minio:
   bucket: '${MINIO_BUCKET:-}'
   prefix_path: '${MINIO_PREFIX_PATH:-}'
 es:
-  hosts: 'http://${ES_HOST:-es01}:9200'
+  hosts: '${ES_HOST:-http://es01:9200}'
   username: '${ES_USER:-elastic}'
   password: '${ELASTIC_PASSWORD:-infini_rag_flow}'
+  api_key: '${ELASTIC_API_KEY:-}'
 os:
   hosts: 'http://${OS_HOST:-opensearch01}:9201'
   username: '${OS_USER:-admin}'

helm/templates/elasticsearch-config.yaml

@@ -1,4 +1,5 @@
 {{- if eq .Values.env.DOC_ENGINE "elasticsearch" -}}
+{{- if .Values.elasticsearch.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -14,4 +15,5 @@ data:
   cluster.routing.allocation.disk.watermark.high: 3gb
   cluster.routing.allocation.disk.watermark.flood_stage: 2gb
   TZ: {{ .Values.env.TZ }}
+{{- end }}
 {{- end -}}

helm/templates/elasticsearch.yaml

@@ -1,4 +1,5 @@
 {{- if eq .Values.env.DOC_ENGINE "elasticsearch" -}}
+{{- if .Values.elasticsearch.enabled }}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -128,4 +129,5 @@ spec:
       port: 9200
       targetPort: http
   type: {{ .Values.elasticsearch.service.type }}
-{{- end -}}
+{{- end }}
+{{- end -}}

helm/templates/env.yaml

@@ -57,8 +57,13 @@ stringData:
   Only provide env vars for enabled doc engine
   */}}
   {{- if eq .Values.env.DOC_ENGINE "elasticsearch" }}
-  ES_HOST: {{ printf "%s-es.%s.svc" (include "ragflow.fullname" .) .Release.Namespace }}
-  ELASTIC_PASSWORD: {{ .Values.env.ELASTIC_PASSWORD | required "ELASTIC_PASSWORD is required" }}
+  {{- if .Values.elasticsearch.enabled }}
+  ES_HOST: {{ printf "http://%s-es.%s.svc" (include "ragflow.fullname" .) .Release.Namespace }}
+  ELASTIC_PASSWORD: {{ .Values.env.ELASTIC_PASSWORD }}
+  {{- else }}
+  ES_HOST: {{ .Values.env.ES_HOST }}
+  ELASTIC_API_KEY: {{ .Values.env.ELASTIC_API_KEY }} 
+  {{- end }}
   {{- else if eq .Values.env.DOC_ENGINE "infinity" }}
   INFINITY_HOST: {{ printf "%s-infinity.%s.svc" (include "ragflow.fullname" .) .Release.Namespace }}
   {{- else if eq .Values.env.DOC_ENGINE "opensearch" }}

helm/values.yaml

@@ -25,6 +25,9 @@ env:
 
   # The password for Elasticsearch
   ELASTIC_PASSWORD: infini_rag_flow_helm
+  # if using elastic-cloud deployment, set enabled: false
+  # ES_HOST: 'https://xxxx'
+  # ELASTIC_API_KEY: replace-if-used-elastic-cloud
 
   # The password for OpenSearch.
   # At least one uppercase letter, one lowercase letter, one digit, and one special character
@@ -130,6 +133,7 @@ infinity:
     type: ClusterIP
 
 elasticsearch:
+  enabled: true
   image:
     repository: elasticsearch
     tag: "8.11.3"