Merge branch 'main' into dependabot/pip/grpcio-1.69.0

Author: xrmx

.github/workflows/ci.yml

@@ -40,6 +40,8 @@ jobs:
     - run: pip-licenses
 
   operator-image-buildable:
+    env:
+      USE_ELASTIC_REGISTRY: ${{ github.event_name != 'pull_request' || ( github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && github.actor != 'dependabot[bot]' ) }}
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
@@ -54,7 +56,11 @@ jobs:
         registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }}
         username: ${{ secrets.ELASTIC_DOCKER_USERNAME }}
         password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }}
+      if: ${{ env.USE_ELASTIC_REGISTRY == 'true' }}
     - run: docker build -f operator/Dockerfile --build-arg DISTRO_DIR=./dist .
+      if: ${{ env.USE_ELASTIC_REGISTRY == 'true' }}
+    - run: docker build -f operator/Dockerfile --build-arg PYTHON_GLIBC_IMAGE=cgr.dev/chainguard/python --build-arg PYTHON_GLIBC_IMAGE_VERSION=latest-dev --build-arg DISTRO_DIR=./dist --build-arg IMAGE=cgr.dev/chainguard/bash --build-arg IMAGE_VERSION=latest .
+      if: ${{ env.USE_ELASTIC_REGISTRY != 'true'}}
 
   test:
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -100,7 +100,7 @@ jobs:
 
       - name: Build and push image
         id: docker-push
-        uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355  # v6.10.0
+        uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc  # v6.11.0
         with:
           context: .
           platforms: linux/amd64,linux/arm64

.gitignore

@@ -24,6 +24,7 @@ coverage
 .eggs
 .cache
 /testdb.sql
+.venv
 venv
 benchmarks/result*
 coverage.xml

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Elastic Distribution of OpenTelemetry Python Changelog
 
+## v0.6.1
+
+- Bump opentelemetry-sdk-extension-aws to 2.1.0 (#222)
+- Bump opentelemetry-resourcedetector-gcp to 1.8.0a0 (#229)
+- Add OpenAI examples (#226)
+
 ## v0.6.0
 
 - Bump to OTel 1.29.0 (#211)

README.md

@@ -80,12 +80,12 @@ This distribution sets the following defaults:
 - `OTEL_METRICS_EXPORTER`: `otlp`
 - `OTEL_LOGS_EXPORTER`: `otlp`
 - `OTEL_EXPORTER_OTLP_PROTOCOL`: `grpc`
-- `OTEL_EXPERIMENTAL_RESOURCE_DETECTORS`: `process_runtime,os,otel,telemetry_distro,aws_ec2,aws_ecs,aws_elastic_beanstalk,azure_app_service,azure_vm`
+- `OTEL_EXPERIMENTAL_RESOURCE_DETECTORS`: `process_runtime,os,otel,telemetry_distro,_gcp,aws_ec2,aws_ecs,aws_elastic_beanstalk,azure_app_service,azure_vm`
 - `OTEL_METRICS_EXEMPLAR_FILTER`: `always_off`
 - `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE`: `DELTA`
 
 > [!NOTE]
-> `OTEL_EXPERIMENTAL_RESOURCE_DETECTORS` cloud resource detectors are dynamically set. When running in a Kubernetes Pod it will be set to `process_runtime,os,otel,telemetry_distro,aws_eks`.
+> `OTEL_EXPERIMENTAL_RESOURCE_DETECTORS` cloud resource detectors are dynamically set. When running in a Kubernetes Pod it will be set to `process_runtime,os,otel,telemetry_distro,_gcp,aws_eks`.
 
 ### Distribution specific configuration variables
 

dev-requirements.txt

@@ -71,7 +71,7 @@ opentelemetry-proto==1.29.0
     #   oteltest
 opentelemetry-resource-detector-azure==0.1.5
     # via elastic-opentelemetry (pyproject.toml)
-opentelemetry-resourcedetector-gcp==1.7.0a0
+opentelemetry-resourcedetector-gcp==1.8.0a0
     # via elastic-opentelemetry (pyproject.toml)
 opentelemetry-sdk==1.29.0
     # via
@@ -100,7 +100,7 @@ pip-tools==7.4.1
     # via elastic-opentelemetry (pyproject.toml)
 pluggy==1.5.0
     # via pytest
-protobuf==5.29.2
+protobuf==5.29.3
     # via
     #   googleapis-common-protos
     #   opentelemetry-proto
@@ -130,7 +130,7 @@ urllib3==2.2.3
     # via requests
 wheel==0.45.1
     # via pip-tools
-wrapt==1.17.0
+wrapt==1.17.1
     # via
     #   deprecated
     #   opentelemetry-instrumentation

examples/openai/README.md

@@ -0,0 +1,77 @@
+# OpenAI Zero-Code Instrumentation Examples
+
+This is an example of how to instrument OpenAI calls with zero code changes,
+using `opentelemetry-instrument` included in the Elastic Distribution of
+OpenTelemetry Python ([EDOT Python][edot-python]).
+
+When OpenAI examples run, they export traces, metrics and logs to an OTLP
+compatible endpoint. Traces and metrics include details such as the model used
+and the duration of the LLM request. In the case of chat, Logs capture the
+request and the generated response. The combination of these provide a
+comprehensive view of the performance and behavior of your OpenAI usage.
+
+## Install
+
+First, set up a Python virtual environment like this:
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+Next, install [EDOT Python][edot-python] and dotenv which is a portable way to
+load environment variables.
+```bash
+pip install "python-dotenv[cli]" elastic-opentelemetry
+```
+
+Finally, run `edot-bootstrap` which analyzes the code to add relevant
+instrumentation, to record traces, metrics and logs.
+```bash
+edot-bootstrap --action=install
+```
+
+## Configure
+
+Copy [env.example](env.example) to `.env` and update its `OPENAI_API_KEY`.
+
+An OTLP compatible endpoint should be listening for traces, metrics and logs on
+`http://localhost:4317`. If not, update `OTEL_EXPORTER_OTLP_ENDPOINT` as well.
+
+For example, if Elastic APM server is running locally, edit `.env` like this:
+```
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200
+```
+
+## Run
+
+There are two examples, and they run the same way:
+
+### Chat
+
+[chat.py](chat.py) asks the LLM a geography question and prints the response.
+
+Run it like this:
+```bash
+dotenv run -- opentelemetry-instrument python chat.py
+```
+
+You should see something like "Atlantic Ocean" unless your LLM hallucinates!
+
+### Embeddings
+
+
+[embeddings.py](embeddings.py) creates in-memory VectorDB embeddings about
+Elastic products. Then, it searches for one similar to a question.
+
+Run it like this:
+```bash
+dotenv run -- opentelemetry-instrument python embeddings.py
+```
+
+You should see something like "Connectors can help you connect to a database",
+unless your LLM hallucinates!
+
+---
+
+[edot-python]: https://github.com/elastic/elastic-otel-python/blob/main/docs/get-started.md

examples/openai/chat.py

@@ -0,0 +1,39 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import openai
+
+CHAT_MODEL = os.environ.get("CHAT_MODEL", "gpt-4o-mini")
+
+
+def main():
+    client = openai.Client()
+
+    messages = [
+        {
+            "role": "user",
+            "content": "Answer in up to 3 words: Which ocean contains Bouvet Island?",
+        }
+    ]
+
+    chat_completion = client.chat.completions.create(model=CHAT_MODEL, messages=messages)
+    print(chat_completion.choices[0].message.content)
+
+
+if __name__ == "__main__":
+    main()

examples/openai/embeddings.py

@@ -0,0 +1,65 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import numpy as np
+import openai
+
+EMBEDDINGS_MODEL = os.environ.get("EMBEDDINGS_MODEL", "text-embedding-3-small")
+
+
+def main():
+    client = openai.Client()
+
+    products = [
+        "Search: Ingest your data, and explore Elastic's machine learning and retrieval augmented generation (RAG) capabilities."
+        "Observability: Unify your logs, metrics, traces, and profiling at scale in a single platform.",
+        "Security: Protect, investigate, and respond to cyber threats with AI-driven security analytics."
+        "Elasticsearch: Distributed, RESTful search and analytics.",
+        "Kibana: Visualize your data. Navigate the Stack.",
+        "Beats: Collect, parse, and ship in a lightweight fashion.",
+        "Connectors: Connect popular databases, file systems, collaboration tools, and more.",
+        "Logstash: Ingest, transform, enrich, and output.",
+    ]
+
+    # Generate embeddings for each product. Keep them in an array instead of a vector DB.
+    product_embeddings = []
+    for product in products:
+        product_embeddings.append(create_embedding(client, product))
+
+    query_embedding = create_embedding(client, "What can help me connect to a database?")
+
+    # Calculate cosine similarity between the query and document embeddings
+    similarities = []
+    for product_embedding in product_embeddings:
+        similarity = np.dot(query_embedding, product_embedding) / (
+            np.linalg.norm(query_embedding) * np.linalg.norm(product_embedding)
+        )
+        similarities.append(similarity)
+
+    # Get the index of the most similar document
+    most_similar_index = np.argmax(similarities)
+
+    print(products[most_similar_index])
+
+
+def create_embedding(client, text):
+    return client.embeddings.create(input=[text], model=EMBEDDINGS_MODEL, encoding_format="float").data[0].embedding
+
+
+if __name__ == "__main__":
+    main()

examples/openai/env.example

@@ -0,0 +1,26 @@
+# Update this with your real OpenAI API key
+OPENAI_API_KEY=sk-YOUR_API_KEY
+
+# Uncomment to use Ollama instead of OpenAI
+# OPENAI_BASE_URL=http://localhost:11434/v1
+# OPENAI_API_KEY=unused
+# CHAT_MODEL=qwen2.5:0.5b
+# EMBEDDINGS_MODEL=all-minilm:33m
+
+# OTEL_EXPORTER_* variables are not required. If you would like to change your
+# OTLP endpoint to Elastic APM server using HTTP, uncomment the following:
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200
+# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+
+OTEL_SERVICE_NAME=openai-example
+
+# Change to 'false' to hide prompt and completion content
+OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true
+# Change to affect behavior of which resources are detected. Note: these
+# choices are specific to the language, in this case Python.
+OTEL_EXPERIMENTAL_RESOURCE_DETECTORS=process_runtime,os,otel,telemetry_distro
+
+# Export metrics every 3 seconds instead of every minute
+OTEL_METRIC_EXPORT_INTERVAL=3000
+# Export traces every 3 seconds instead of every 5 seconds
+OTEL_BSP_SCHEDULE_DELAY=3000