diff --git a/.github/workflows/python-ec2-genai-test.yml b/.github/workflows/python-ec2-genai-test.yml new file mode 100644 index 000000000..07b6b2a4f --- /dev/null +++ b/.github/workflows/python-ec2-genai-test.yml @@ -0,0 +1,163 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +name: Python EC2 Gen AI Use Case +on: + workflow_dispatch: # be able to run the workflow on demand + + workflow_call: + inputs: + caller-workflow-name: + required: true + type: string + staging-wheel-name: + required: false + default: 'aws-opentelemetry-distro' + type: string + +permissions: + id-token: write + contents: read + +env: + E2E_TEST_AWS_REGION: 'us-west-2' + E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }} + E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }} + ADOT_WHEEL_NAME: ${{ inputs.staging-wheel-name }} + METRIC_NAMESPACE: genesis + LOG_GROUP_NAME: test/genesis + TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE} + SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-us-east-1/python-genai-sample-app.zip + +jobs: + python-ec2-adot-genai: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Set Get ADOT Wheel command environment variable + run: | + if [ "${{ github.event.repository.name }}" = "aws-otel-python-instrumentation" ]; then + # Reusing the adot-main-build-staging-jar bucket to store the python wheel file + echo GET_ADOT_WHEEL_COMMAND="aws s3 cp s3://adot-main-build-staging-jar/${{ env.ADOT_WHEEL_NAME }} ./${{ env.ADOT_WHEEL_NAME }} && python3.12 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV + elif [ "${{ env.OTEL_SOURCE }}" == "pypi" ]; then + echo GET_ADOT_WHEEL_COMMAND="python3.12 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV + else + latest_release_version=$(curl -sL https://github.com/aws-observability/aws-otel-python-instrumentation/releases/latest | grep -oP '/releases/tag/v\K[0-9]+\.[0-9]+\.[0-9]+' | head -n 1) + echo "The latest version is $latest_release_version" + echo GET_ADOT_WHEEL_COMMAND="wget -O ${{ env.ADOT_WHEEL_NAME }} https://github.com/aws-observability/aws-otel-python-instrumentation/releases/latest/download/aws_opentelemetry_distro-$latest_release_version-py3-none-any.whl \ + && python3.12 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV + fi + + - name: Initiate Gradlew Daemon + uses: ./.github/workflows/actions/execute_and_retry + continue-on-error: true + with: + command: "./gradlew :validator:build" + cleanup: "./gradlew clean" + max_retry: 3 + sleep_time: 60 + + - name: Generate testing id + run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}-${RANDOM}" >> $GITHUB_ENV + + - name: Generate XRay and W3C trace ID + run: | + ID_1="$(printf '%08x' $(date +%s))" + ID_2="$(openssl rand -hex 12)" + W3C_TRACE_ID="${ID_1}${ID_2}" + XRAY_TRACE_ID="1-${ID_1}-${ID_2}" + PARENT_ID="$(openssl rand -hex 8)" + TRACE_ID_HEADER="Root=${XRAY_TRACE_ID};Parent=${PARENT_ID};Sampled=1" + echo "XRAY_TRACE_ID=${XRAY_TRACE_ID}" >> $GITHUB_ENV + echo "W3C_TRACE_ID=${W3C_TRACE_ID}" >> $GITHUB_ENV + echo "TRACE_ID_HEADER=${TRACE_ID_HEADER}" >> $GITHUB_ENV + echo "Generated XRay Trace ID: ${XRAY_TRACE_ID}" + echo "Generated W3C Trace ID: ${W3C_TRACE_ID}" + echo "Generated Trace ID Header: ${TRACE_ID_HEADER}" + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.E2E_TEST_ACCOUNT_ID }}:role/${{ env.E2E_TEST_ROLE_NAME }} + aws-region: ${{ env.E2E_TEST_AWS_REGION }} + + - name: Set up terraform + uses: ./.github/workflows/actions/execute_and_retry + with: + command: "wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg" + post-command: 'echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + && sudo apt update && sudo apt install terraform' + + - name: Initiate Terraform + uses: ./.github/workflows/actions/execute_and_retry + with: + command: "cd ${{ env.TEST_RESOURCES_FOLDER }}/terraform/python/ec2/adot-genai && terraform init && terraform validate" + cleanup: "rm -rf .terraform && rm -rf .terraform.lock.hcl" + max_retry: 6 + + - name: Deploy service via terraform + working-directory: terraform/python/ec2/adot-genai + run: | + terraform apply -auto-approve \ + -var="aws_region=${{ env.E2E_TEST_AWS_REGION }}" \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="service_zip_url=${{ env.SAMPLE_APP_ZIP }}" \ + -var="trace_id=${{ env.TRACE_ID_HEADER }}" \ + -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \ + + + - name: Get deployment info + working-directory: terraform/python/ec2/adot-genai + run: | + echo "INSTANCE_IP=$(terraform output langchain_service_public_ip)" >> $GITHUB_ENV + echo "INSTANCE_ID=$(terraform output langchain_service_instance_id)" >> $GITHUB_ENV + + - name: Waiting 5 Minutes for Gen AI service to be ready and emit logs, traces, and metrics + run: sleep 300 + + - name: Validate generated logs + run: ./gradlew validator:run --args='-c python/ec2/adot-genai/log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.INSTANCE_IP }}:8000 + --region ${{ env.E2E_TEST_AWS_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name langchain-traceloop-app + --instance-id ${{ env.INSTANCE_ID }} + --trace-id ${{ env.W3C_TRACE_ID }}' + + - name: Validate generated traces + if: (success() || failure()) && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/adot-genai/trace-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.INSTANCE_IP }}:8000 + --region ${{ env.E2E_TEST_AWS_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --service-name langchain-traceloop-app + --instance-id ${{ env.INSTANCE_ID }} + --trace-id ${{ env.XRAY_TRACE_ID }}' + + - name: Validate generated metrics + if: (success() || failure()) && !cancelled() + run: ./gradlew validator:run --args='-c python/ec2/adot-genai/metric-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.INSTANCE_IP }}:8000 + --region ${{ env.E2E_TEST_AWS_REGION }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --log-group ${{ env.LOG_GROUP_NAME }} + --service-name langchain-traceloop-app + --instance-id ${{ env.INSTANCE_ID }}' + + - name: Cleanup + if: always() + continue-on-error: true + working-directory: terraform/python/ec2/adot-genai + run: | + terraform destroy -auto-approve \ + -var="aws_region=${{ env.E2E_TEST_AWS_REGION }}" \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="service_zip_url=${{ env.SAMPLE_APP_ZIP }}" \ + -var="trace_id=${{ env.TRACE_ID_HEADER }}" \ + -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \ No newline at end of file diff --git a/sample-apps/python/genai_service/ec2-requirements.txt b/sample-apps/python/genai_service/ec2-requirements.txt new file mode 100644 index 000000000..29c31bd1b --- /dev/null +++ b/sample-apps/python/genai_service/ec2-requirements.txt @@ -0,0 +1,15 @@ +langchain +langchain-community +langchain_aws +opentelemetry-sdk +openinference-instrumentation-langchain +opentelemetry-api +opentelemetry-semantic-conventions +python-dotenv +openlit +botocore +setuptools +boto3 +aws_opentelemetry_distro +fastapi +uvicorn[standard] \ No newline at end of file diff --git a/sample-apps/python/genai_service/server.py b/sample-apps/python/genai_service/server.py new file mode 100644 index 000000000..28572c7b7 --- /dev/null +++ b/sample-apps/python/genai_service/server.py @@ -0,0 +1,114 @@ +import os +from typing import Dict, List +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from langchain_aws import ChatBedrock +from langchain.prompts import ChatPromptTemplate +from langchain.chains import LLMChain +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from openinference.instrumentation.langchain import LangChainInstrumentor + +# Load environment variables +load_dotenv() + +# Set up OpenTelemetry with BOTH exporters +tracer_provider = TracerProvider() + +# Add Console exporter +console_exporter = ConsoleSpanExporter() +console_processor = BatchSpanProcessor(console_exporter) +tracer_provider.add_span_processor(console_processor) + +# Add OTLP exporter +otlp_exporter = OTLPSpanExporter(endpoint="http://localhost:4318/v1/traces") +otlp_processor = BatchSpanProcessor(otlp_exporter) +tracer_provider.add_span_processor(otlp_processor) + +# Set as global provider +trace.set_tracer_provider(tracer_provider) + +# Instrument LangChain with OpenInference +LangChainInstrumentor().instrument(tracer_provider=tracer_provider) + +# Initialize FastAPI app +app = FastAPI(title="LangChain Bedrock OpenInference API", version="1.0.0") + +# Initialize the LLM with AWS Bedrock +llm = ChatBedrock( + model_id="anthropic.claude-3-haiku-20240307-v1:0", + model_kwargs={ + "temperature": 0.7, + "max_tokens": 500 + }, + region_name=os.getenv("AWS_DEFAULT_REGION", "us-west-2") +) + +# Create a prompt template +prompt = ChatPromptTemplate.from_template( + "You are a helpful assistant. The user says: {input}. Provide a helpful response." +) + +# Create a chain +chain = LLMChain(llm=llm, prompt=prompt) + +# Request models +class ChatRequest(BaseModel): + message: str + +class BatchChatRequest(BaseModel): + messages: List[str] + +class ChatResponse(BaseModel): + response: str + +class BatchChatResponse(BaseModel): + responses: List[Dict[str, str]] + +# Sample prompts for testing +SAMPLE_PROMPTS = [ + "What is the capital of France?", + "How do I make a cup of coffee?", + "What are the benefits of exercise?", + "Explain quantum computing in simple terms", + "What's the best way to learn programming?" +] + +@app.get("/") +async def root(): + return { + "message": "LangChain Bedrock OpenInference API is running!", + "endpoints": { + "/ai-chat": "Single message chat endpoint", + "/hello": "Simple hello endpoint" + } + } + +@app.post("/ai-chat", response_model=ChatResponse) +async def chat(request: ChatRequest): + """ + Chat endpoint that processes a single user message through AWS Bedrock + """ + try: + # Process the input through the chain + result = await chain.ainvoke({"input": request.message}) + return ChatResponse(response=result["text"]) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +@app.get("/health") +async def health(): + """Health check endpoint""" + return {"status": "healthy", "llm": "AWS Bedrock Claude 3 Haiku"} + +if __name__ == "__main__": + import uvicorn + print("Starting FastAPI server with AWS Bedrock and OpenInference instrumentation...") + print("Make sure AWS credentials are configured") + print("Server will run on http://localhost:8000") + print("API docs available at http://localhost:8000/docs") + + uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file diff --git a/terraform/python/ec2/adot-genai/main.tf b/terraform/python/ec2/adot-genai/main.tf new file mode 100644 index 000000000..a9bf84667 --- /dev/null +++ b/terraform/python/ec2/adot-genai/main.tf @@ -0,0 +1,143 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.aws_region +} + +resource "aws_default_vpc" "default" { + tags = { + Name = "Default VPC" + } +} + +resource "tls_private_key" "ssh_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "aws_key_pair" "aws_ssh_key" { + key_name = "instance_key-${var.test_id}" + public_key = tls_private_key.ssh_key.public_key_openssh +} + +locals { + ssh_key_name = aws_key_pair.aws_ssh_key.key_name + private_key_content = tls_private_key.ssh_key.private_key_pem +} + +data "aws_ami" "ami" { + owners = ["amazon"] + most_recent = true + filter { + name = "name" + values = ["al2023-ami-*-x86_64"] + } + filter { + name = "state" + values = ["available"] + } + filter { + name = "architecture" + values = ["x86_64"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +resource "aws_instance" "main_service_instance" { + ami = data.aws_ami.ami.id + instance_type = "t3.medium" + key_name = local.ssh_key_name + iam_instance_profile = "APP_SIGNALS_EC2_TEST_ROLE" + vpc_security_group_ids = [aws_default_vpc.default.default_security_group_id] + associate_public_ip_address = true + instance_initiated_shutdown_behavior = "terminate" + + metadata_options { + http_tokens = "required" + } + + root_block_device { + volume_size = 30 + } + + user_data = base64encode(<<-EOF +#!/bin/bash +yum update -y +yum install -y python3.12 python3.12-pip unzip bc + +mkdir -p /app +cd /app +aws s3 cp ${var.service_zip_url} genai-service.zip +unzip genai-service.zip + +# Having issues installing dependencies from ec2-requirements.txt as these dependencies are quite large and cause timeouts/memory issues on EC2, manually installing instead +python3.12 -m pip install fastapi uvicorn[standard] --no-cache-dir +python3.12 -m pip install boto3 botocore setuptools --no-cache-dir +python3.12 -m pip install opentelemetry-api opentelemetry-sdk opentelemetry-semantic-conventions --no-cache-dir +python3.12 -m pip install langchain langchain-community langchain_aws --no-cache-dir +python3.12 -m pip install python-dotenv openlit --no-cache-dir +python3.12 -m pip install openinference-instrumentation-langchain --no-cache-dir +${var.get_adot_wheel_command} + +export AWS_REGION=${var.aws_region} +export OTEL_PROPAGATORS=tracecontext,xray,baggage +export OTEL_PYTHON_DISTRO=aws_distro +export OTEL_PYTHON_CONFIGURATOR=aws_configurator +export OTEL_EXPORTER_OTLP_LOGS_HEADERS="x-aws-log-group=test/genesis,x-aws-log-stream=default,x-aws-metric-namespace=genesis" +export OTEL_RESOURCE_ATTRIBUTES="service.name=langchain-traceloop-app" +export AGENT_OBSERVABILITY_ENABLED="true" + +nohup opentelemetry-instrument python3.12 server.py > /var/log/langchain-service.log 2>&1 & + +# Wait for service to be ready +echo "Waiting for service to be ready..." +for i in {1..60}; do + if curl -s http://localhost:8000/health > /dev/null 2>&1; then + echo "Service is ready!" + break + fi + echo "Attempt $i: Service not ready, waiting 5 seconds..." + sleep 5 +done + +# Generate traffic directly +echo "Starting traffic generator..." +nohup bash -c ' +for i in {1..5}; do + message="What is the weather like today?" + echo "[$(date)] Request $i: $message" + curl -s -X POST http://localhost:8000/ai-chat \ + -H "Content-Type: application/json" \ + -H "X-Amzn-Trace-Id: ${var.trace_id}" \ + -d "{\"message\": \"$message\"}" \ + -m 30 \ + echo "Request $i completed" + sleep 10 +done +echo "Traffic generator completed" +' > /var/log/traffic-generator.log 2>&1 & +EOF + ) + + tags = { + Name = "langchain-service-${var.test_id}" + } +} + +output "langchain_service_instance_id" { + value = aws_instance.main_service_instance.id +} + +output "langchain_service_public_ip" { + value = aws_instance.main_service_instance.public_ip +} \ No newline at end of file diff --git a/terraform/python/ec2/adot-genai/variables.tf b/terraform/python/ec2/adot-genai/variables.tf new file mode 100644 index 000000000..b8f0d21cb --- /dev/null +++ b/terraform/python/ec2/adot-genai/variables.tf @@ -0,0 +1,44 @@ +# ------------------------------------------------------------------------ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# ------------------------------------------------------------------------- + +variable "aws_region" { + default = "us-west-2" +} + +variable "test_id" { + default = "dummy-123" +} + +variable "service_zip_url" { + description = "S3 URL for the service zip file" +} + + + + + +variable "user" { + default = "ec2-user" +} + +variable "trace_id" { + description = "Trace ID for X-Ray tracing" + default = "Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=1" +} + +variable "get_adot_wheel_command" { + description = "Command to get and install ADOT wheel" + default = "python3.12 -m pip install aws-opentelemetry-distro" +} \ No newline at end of file