Merge branch 'elastic:main' into main

Author: kritinv

bin/find-notebooks-to-test.sh

@@ -30,6 +30,8 @@ EXEMPT_NOTEBOOKS=(
     "notebooks/integrations/llama3/rag-elastic-llama3.ipynb"
     "notebooks/integrations/azure-openai/vector-search-azure-openai-elastic.ipynb"
     "notebooks/enterprise-search/app-search-engine-exporter.ipynb",
+    "notebooks/enterprise-search/elastic-crawler-to-open-crawler-migration.ipynb",
+    "notebooks/enterprise-search/app-search-crawler-to-open-crawler-migration.ipynb",
     "notebooks/playground-examples/bedrock-anthropic-elasticsearch-client.ipynb",
     "notebooks/playground-examples/openai-elasticsearch-client.ipynb",
     "notebooks/integrations/hugging-face/huggingface-integration-millions-of-documents-with-cohere-reranking.ipynb",

docker/README.md

@@ -9,8 +9,11 @@ Note: If you haven't checked out this repository, all you need is one file:
 wget https://raw.githubusercontent.com/elastic/elasticsearch-labs/refs/heads/main/docker/docker-compose-elastic.yml
 ```
 
-Use docker compose to run Elastic stack in the background:
+Before you begin, ensure you have free CPU and memory on your Docker host. If
+you plan to use ELSER, assume a minimum of 8 cpus and 6GB memory for the
+containers in this compose file.
 
+First, start this Elastic Stack in the background:
 ```bash
 docker compose -f docker-compose-elastic.yml up --force-recreate --wait -d
 ```
@@ -20,7 +23,6 @@ Then, you can view Kibana at http://localhost:5601/app/home#/
 If asked for a username and password, use username: elastic and password: elastic.
 
 Clean up when finished, like this:
-
 ```bash
 docker compose -f docker-compose-elastic.yml down
 ```

docker/docker-compose-elastic.yml

@@ -27,7 +27,7 @@ services:
       test:  # readiness probe taken from kbn-health-gateway-server script
         [
           "CMD-SHELL",
-          "curl -s http://localhost:9200 | grep -q 'missing authentication credentials'",
+          "curl --max-time 1 -s http://localhost:9200 | grep -q 'missing authentication credentials'",
         ]
       start_period: 10s
       interval: 1s
@@ -41,12 +41,15 @@ services:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.17.2
     container_name: elasticsearch_settings
     restart: 'no'
+    # gen-ai assistants in kibana save state in a way that requires system
+    # access, so set kibana_system's password to a known value.
     command: >
-      bash -c '        
-        # gen-ai assistants in kibana save state in a way that requires security to be enabled, so we need to create
-        # a kibana system user before starting it.
+      bash -c '
         echo "Setup the kibana_system password";
-        until curl -s -u "elastic:elastic" -X POST http://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"elastic\"}" -H "Content-Type: application/json" | grep -q "^{}"; do sleep 5; done;
+        until curl --max-time 1 -s -u "elastic:elastic" \
+          -X POST http://elasticsearch:9200/_security/user/kibana_system/_password \
+          -d "{\"password\":\"elastic\"}" \
+          -H "Content-Type: application/json" | grep -q "^{}"; do sleep 5; done;
       '
 
   kibana:
@@ -69,7 +72,7 @@ services:
       - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=fhjskloppd678ehkdfdlliverpoolfcr
       - SERVER_PUBLICBASEURL=http://127.0.0.1:5601
     healthcheck:
-      test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status | grep -q 'All services are available'"]
+      test: ["CMD-SHELL", "curl --max-time 1 -s http://localhost:5601/api/status | grep -q 'available'"]
       retries: 300
       interval: 1s
 

example-apps/chatbot-rag-app/Dockerfile

@@ -5,24 +5,17 @@ COPY frontend ./frontend
 RUN cd frontend && yarn install
 RUN cd frontend && REACT_APP_API_HOST=/api yarn build
 
+# Use glibc-based image to get pre-compiled wheels for grpcio and tiktoken
 FROM python:3.12-slim
 
 WORKDIR /app
 RUN mkdir -p ./frontend/build
 COPY --from=build-step ./app/frontend/build ./frontend/build
-RUN mkdir ./api
-RUN mkdir ./data
-
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    curl \
-    software-properties-common \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
 
 COPY requirements.txt ./requirements.txt
 RUN pip3 install -r ./requirements.txt
+
+RUN mkdir -p ./api ./data
 COPY api ./api
 COPY data ./data
 
@@ -31,16 +24,6 @@ EXPOSE 4000
 # Default to disabling instrumentation, can be overridden to false in
 # docker invocations to reenable.
 ENV OTEL_SDK_DISABLED=true
+ENTRYPOINT [ "opentelemetry-instrument" ]
 
-# https://github.com/elastic/genai-instrumentation/issues/255
-# Currently Python SDK has a bug that spams logs when opentelemetry-instrument is used
-# with SDK being disabled. Until it is fixed, we handle it in our own entrypoint by
-# avoiding opentelemetry-instrument when SDK is disabled.
-RUN echo 'if [ "${OTEL_SDK_DISABLED:-true}" == "false" ]; \
-  then \
-    opentelemetry-instrument $@; \
-  else \
-    exec $@; \
-  fi' > entrypoint.sh
-ENTRYPOINT [ "bash", "-eu", "./entrypoint.sh" ]
-CMD [ "python", "api/app.py"]
+CMD [ "python", "api/app.py" ]

example-apps/chatbot-rag-app/README.md

@@ -22,8 +22,8 @@ Copy [env.example](env.example) to `.env` and fill in values noted inside.
 ## Installing and connecting to Elasticsearch
 
 There are a number of ways to install Elasticsearch. Cloud is best for most
-use-cases. We also have [docker-compose-elastic.yml](../../docker), that starts
-Elasticsearch, Kibana, and APM Server on your laptop with one command.
+use-cases. We also have [docker-compose-elastic.yml][docker-compose-elastic],
+that starts Elasticsearch, Kibana, and APM Server on your laptop in one step.
 
 Once you decided your approach, edit your `.env` file accordingly.
 
@@ -71,6 +71,69 @@ Clean up when finished, like this:
 docker compose down
 ```
 
+### Run with Kubernetes
+
+Kubernetes is more complicated than Docker, but closer to the production
+experience for many users. [k8s-manifest.yml](k8s-manifest.yml) creates the
+same services, but needs additional configuration first.
+
+First step is to setup your environment. [env.example](env.example) must be
+copied to a file name `.env` and updated with `ELASTICSEARCH_URL` and
+`OTEL_EXPORTER_OTLP_ENDPOINT` values visible to you Kubernetes deployment.
+
+For example, if you started your Elastic Stack with [k8s-manifest-elastic.yml][k8s-manifest-elastic],
+you would update these values:
+```
+ELASTICSEARCH_URL=http://elasticsearch:9200
+OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200
+```
+
+Then, import your `.env` file as a configmap like this:
+```bash
+kubectl create configmap chatbot-rag-app-env --from-env-file=.env
+```
+
+<details>
+<summary>To use Vertex AI, set `LLM_TYPE=vertex` in your `.env` and follow these steps</summary>
+
+The `api-frontend container` needs access to your Google Cloud credentials.
+Share your `application_default_credentials.json` as a Kubernetes secret:
+```bash
+# Logs you into Google Cloud and creates application_default_credentials.json
+gcloud auth application-default login
+# Adds your credentials to a Kubernetes secret named gcloud-credentials
+kubectl create secret generic gcloud-credentials \
+  --from-file=application_default_credentials.json=$HOME/.config/gcloud/application_default_credentials.json
+```
+</details>
+
+Now that your configuration is applied, create the `chatbot-rag-app` deployment
+and service by applying this manifest:
+```bash
+kubectl apply -f k8s-manifest.yml
+```
+
+Next, block until `chatbot-rag-app` is available.
+```bash
+kubectl wait --for=condition=available --timeout=20m  deployment/chatbot-rag-app
+```
+
+*Note*: The first run may take several minutes to become available. Here's how
+to follow logs on this stage:
+```bash
+kubectl logs deployment.apps/chatbot-rag-app -c create-index -f
+```
+
+Next, forward the web UI port:
+```bash
+kubectl port-forward deployment.apps/chatbot-rag-app 4000:4000 &
+```
+
+Clean up when finished, like this:
+```bash
+kubectl delete -f k8s-manifest.yml
+```
+
 ### Run with Python
 
 If you want to run this example with Python, you need to do a few things listed
@@ -165,8 +228,6 @@ pip-compile
 pip install -r requirements.txt
 # Add opentelemetry instrumentation for these dependencies
 edot-bootstrap >> requirements.txt
-# Missing dependency for langtrace vertexai instrumentation
-echo "setuptools" >> requirements.txt
 # Install opentelemetry dependencies
 pip install -r requirements.txt
 ```
@@ -198,3 +259,5 @@ docker compose up --build --force-recreate
 ---
 [loader-docs]: https://python.langchain.com/docs/how_to/#document-loaders
 [install-es]: https://www.elastic.co/search-labs/tutorials/install-elasticsearch
+[docker-compose-elastic]: ../../docker/docker-compose-elastic.yml
+[k8s-manifest-elastic]: ../../k8s/k8s-manifest-elastic.yml

example-apps/chatbot-rag-app/env.example

@@ -6,6 +6,10 @@ FLASK_APP=api/app.py
 PYTHONUNBUFFERED=1
 
 # How you connect to Elasticsearch: change details to your instance
+# This defaults to a Elastic Stack accessible via localhost.
+#
+# When running inside Kubernetes, set to http://elasticsearch.default.svc:9200
+# or similar.
 ELASTICSEARCH_URL=http://localhost:9200
 ELASTICSEARCH_USER=elastic
 ELASTICSEARCH_PASSWORD=elastic
@@ -68,7 +72,11 @@ OTEL_SDK_DISABLED=true
 # Assign the service name that shows up in Kibana
 OTEL_SERVICE_NAME=chatbot-rag-app
 
-# Default to send traces to the Elastic APM server
+# Default to send logs, traces and metrics to an Elastic APM server accessible
+# via localhost.
+#
+# When running inside Kubernetes, set to http://elasticsearch.default.svc:9200
+# or similar.
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200
 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
 

example-apps/chatbot-rag-app/k8s-manifest.yml

@@ -0,0 +1,58 @@
+---
+# chatbot-rag-app deploys "create-index" to install ELSER and load values.
+# Then, it starts "api-frontend" to serve the application.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: chatbot-rag-app
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: chatbot-rag-app
+  template:
+    metadata:
+      labels:
+        app: chatbot-rag-app
+    spec:
+      # For `LLM_TYPE=vertex`: create a volume for application_default_credentials.json
+      volumes:
+        - name: gcloud-credentials
+          secret:
+            secretName: gcloud-credentials
+            optional: true  # only read when `LLM_TYPE=vertex`
+      initContainers:
+        - name: create-index
+          image: &image ghcr.io/elastic/elasticsearch-labs/chatbot-rag-app:latest
+          command: &command [ "opentelemetry-instrument" ] # match image
+          args: [ "flask", "create-index" ]
+          # This recreates your configmap based on your .env file:
+          # kubectl create configmap chatbot-rag-app-env --from-env-file=.env
+          envFrom: &envFrom
+            - configMapRef:
+                name: chatbot-rag-app-env
+      containers:
+        - name: api-frontend
+          image: *image
+          command: *command
+          args: [ "python", "api/app.py" ]
+          ports:
+            - containerPort: 4000
+          envFrom: *envFrom
+          # For `LLM_TYPE=vertex`: mount credentials to the path read by the google-cloud-sdk
+          volumeMounts:
+            - name: gcloud-credentials
+              mountPath: /root/.config/gcloud
+              readOnly: true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: api
+spec:
+  selector:
+    app: chatbot-rag-app
+  ports:
+    - protocol: TCP
+      port: 4000
+      targetPort: 4000

example-apps/chatbot-rag-app/requirements.in

@@ -16,5 +16,5 @@ langchain-mistralai
 
 # EDOT dependencies
 elastic-opentelemetry
-# Additional LLM support not in EDOT
+# Additional LLM support not yet in EDOT
 langtrace-python-sdk