Skip to content

Commit 1f40d7a

Browse files
alvaroloesalvaroloes
authored
Merge branch 'main' into run-evaluations-from-ui
2 parents 8e6360c + 176532e commit 1f40d7a

File tree

6 files changed

+120
-17
lines changed

6 files changed

+120
-17
lines changed

‎.github/workflows/app-build.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@ env:
2121
GIT_REF: ${{ github.ref }} # GitHub Context
2222
FRONTEND_BUILD_CONTEXT: ./apps/opik-frontend # Repository VAR
2323
BACKEND_BUILD_CONTEXT: ./apps/opik-backend # Repository VAR
24+
EVALUATOR_BUILD_CONTEXT: ./apps/aifindr-evaluations-runner # Repository VAR
2425
ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} # Repository VAR
2526
ECR_BACKEND_REPOSITORY: ${{ vars.ECR_BACKEND_REPOSITORY }} # Repository VAR
2627
ECR_FRONTEND_REPOSITORY: ${{ vars.ECR_FRONTEND_REPOSITORY }} # Repository VAR
28+
ECR_EVALUATOR_REPOSITORY: ${{ vars.ECR_EVALUATOR_REPOSITORY }} # Repository VAR
2729

2830
jobs:
2931
# Check if latest tag is a release candidate and stop flow if it is
@@ -154,6 +156,18 @@ jobs:
154156
docker push $ECR_REGISTRY/$ECR_BACKEND_REPOSITORY:$APP_VERSION
155157
docker push $ECR_REGISTRY/$ECR_BACKEND_REPOSITORY:$IMAGE_TAG
156158
159+
if [ $DEBUG == "true" ]; then
160+
echo "building evaluator image into: $ECR_REGISTRY/$ECR_EVALUATOR_REPOSITORY:$APP_VERSION"
161+
fi
162+
163+
docker build -t $ECR_REGISTRY/$ECR_EVALUATOR_REPOSITORY:$APP_VERSION \
164+
-f $EVALUATOR_BUILD_CONTEXT/Dockerfile \
165+
$EVALUATOR_BUILD_CONTEXT
166+
167+
docker tag $ECR_REGISTRY/$ECR_EVALUATOR_REPOSITORY:$APP_VERSION $ECR_REGISTRY/$ECR_EVALUATOR_REPOSITORY:$IMAGE_TAG
168+
docker push $ECR_REGISTRY/$ECR_EVALUATOR_REPOSITORY:$APP_VERSION
169+
docker push $ECR_REGISTRY/$ECR_EVALUATOR_REPOSITORY:$IMAGE_TAG
170+
157171
ecs-deploy:
158172
environment: "${{ inputs.environment || 'production' }}"
159173
env:

‎apps/aifindr-evaluations-runner/.env.example‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
1-
OPIK_URL=http://host.docker.internal:5173/api
2-
OPENAI_API_KEY=your-api-key-here
1+
OPIK_URL_OVERRIDE=http://host.docker.internal:5173/api
2+
OPENAI_API_KEY=your-api-key-here
3+
ELLMENTAL_API_URL=
4+
ELLMENTAL_API_KEY=

‎apps/aifindr-evaluations-runner/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM python:3.11-slim
1+
FROM python:3.12-slim
22

33
WORKDIR /app
44

‎apps/aifindr-evaluations-runner/main.py‎

Lines changed: 79 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@
33
import logging
44
import asyncio
55
import uuid
6+
from requests import request, Response
7+
8+
from urllib.parse import urljoin, urlparse
9+
from typing import Dict
10+
11+
from settings import settings
612
from evaluator import EvaluationParams, ExperimentStatus, execute_evaluation
713

814
# Configure logging
@@ -11,9 +17,12 @@
1117

1218
app = FastAPI()
1319

14-
TASK_QUEUE: asyncio.Queue[EvaluationParams] = asyncio.Queue(maxsize=10) # Maximum number of evaluations in queue
20+
TASK_QUEUE: asyncio.Queue[EvaluationParams] = asyncio.Queue(
21+
maxsize=10
22+
) # Maximum number of evaluations in queue
1523
MAX_CONCURRENT_TASKS = 5 # Number of concurrent evaluations
1624

25+
1726
class RunEvaluationsRequest(BaseModel):
1827
workspace_name: str
1928
dataset_name: str
@@ -22,10 +31,21 @@ class RunEvaluationsRequest(BaseModel):
2231
base_prompt_name: str
2332
workflow: str
2433

34+
2535
class RunEvaluationsResponse(BaseModel):
2636
status: str
2737
task_id: str
2838

39+
40+
class HealthResponse(BaseModel):
41+
ellmental: Dict[str, str]
42+
opik: Dict[str, str]
43+
44+
45+
def get_domain(url: str) -> str:
46+
return urlparse(url).netloc.split(":")[0]
47+
48+
2949
async def process_queue():
3050
"""Background task to process queued evaluations"""
3151
while True:
@@ -40,12 +60,58 @@ async def process_queue():
4060
finally:
4161
TASK_QUEUE.task_done()
4262

63+
4364
@app.on_event("startup")
4465
async def startup_event():
4566
# Start background workers to process the queue
4667
for _ in range(MAX_CONCURRENT_TASKS):
4768
asyncio.create_task(process_queue())
4869

70+
71+
@app.get("/health", response_model=HealthResponse)
72+
async def health(timeout: int = 5):
73+
ellmental_health_url: str = urljoin(settings.ELLMENTAL_API_URL, "/health")
74+
opik_health_url: str = urljoin(settings.OPIK_URL_OVERRIDE, "/is-alive/ping")
75+
76+
ellm_health = {"status": "healthy", "message": "eLLMental is healthy"}
77+
opik_health = {"status": "healthy", "message": "OPIK is healthy"}
78+
try:
79+
ellm_response: Response = request(
80+
method="GET", url=ellmental_health_url, timeout=timeout
81+
)
82+
if ellm_response.status_code != 200:
83+
ellm_health["status"] = "unhealthy"
84+
ellm_health["message"] = str(ellm_response.text).replace(
85+
get_domain(settings.ELLMENTAL_API_URL), "***"
86+
)
87+
except Exception as e:
88+
ellm_health["status"] = "unhealthy"
89+
ellm_health["message"] = str(e).replace(
90+
get_domain(settings.ELLMENTAL_API_URL), "***"
91+
)
92+
try:
93+
opik_response: Response = request(
94+
method="GET", url=opik_health_url, timeout=timeout
95+
)
96+
if opik_response.status_code != 200:
97+
opik_health["status"] = "unhealthy"
98+
opik_health["message"] = str(opik_response.text).replace(
99+
get_domain(settings.OPIK_URL_OVERRIDE), "***"
100+
)
101+
except Exception as e:
102+
opik_health["status"] = "unhealthy"
103+
opik_health["message"] = str(e).replace(
104+
get_domain(settings.OPIK_URL_OVERRIDE), "***"
105+
)
106+
107+
response = {"ellmental": ellm_health, "opik": opik_health}
108+
109+
if all(health["status"] == "healthy" for health in [ellm_health, opik_health]):
110+
return HealthResponse(**response)
111+
else:
112+
raise HTTPException(status_code=503, detail=response)
113+
114+
49115
@app.post("/evaluations/run", response_model=RunEvaluationsResponse)
50116
async def run_evaluation(input: RunEvaluationsRequest, req: Request):
51117
try:
@@ -67,11 +133,17 @@ async def run_evaluation(input: RunEvaluationsRequest, req: Request):
67133
TASK_QUEUE.put_nowait(evaluation_params)
68134
logger.info("Evaluation task added to queue")
69135
except asyncio.QueueFull:
70-
logger.error(f"Queue is full. Evaluation task not added to the queue: {evaluation_params}")
71-
raise HTTPException(status_code=503, detail="Server is currently at maximum capacity. Please try again later.")
72-
73-
return RunEvaluationsResponse(status=ExperimentStatus.RUNNING.value, task_id=task_id)
136+
logger.error(
137+
f"Queue is full. Evaluation task not added to the queue: {evaluation_params}"
138+
)
139+
raise HTTPException(
140+
status_code=503,
141+
detail="Server is currently at maximum capacity. Please try again later.",
142+
)
143+
144+
return RunEvaluationsResponse(
145+
status=ExperimentStatus.RUNNING.value, task_id=task_id
146+
)
74147
except Exception as e:
75148
logger.error(f"Error processing request: {str(e)}")
76-
raise HTTPException(status_code=500, detail=str(e))
77-
149+
raise HTTPException(status_code=500, detail=str(e))

‎apps/aifindr-evaluations-runner/settings.py‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,33 @@
1-
from pydantic_settings import BaseSettings
1+
import os
2+
from functools import lru_cache
3+
4+
from pydantic_settings import BaseSettings, SettingsConfigDict
25
from pydantic import field_validator
36

7+
48
class Settings(BaseSettings):
9+
model_config = SettingsConfigDict(env_file=".env", extra="allow")
510
OPIK_URL_OVERRIDE: str = "http://host.docker.internal:5173/api"
611
OPENAI_API_KEY: str = ""
712
ELLMENTAL_API_URL: str = ""
813
ELLMENTAL_API_KEY: str = ""
9-
14+
1015
@field_validator("*")
1116
def no_empty_strings(cls, v):
1217
if isinstance(v, str) and not v:
1318
raise ValueError("Field cannot be empty")
1419
return v
15-
16-
class Config:
17-
env_file = ".env"
18-
extra = "ignore" # Permite ignorar variables extra
1920

20-
settings = Settings()
21+
22+
class EnvSettings(BaseSettings):
23+
settings: Settings = Settings()
24+
25+
26+
@lru_cache() # Cache settings to avoid re-reading the .env file on each call
27+
def get_settings() -> Settings:
28+
if not os.getenv("SETTINGS"):
29+
return Settings()
30+
return EnvSettings().settings
31+
32+
33+
settings = get_settings()

‎deployment/docker-compose/docker-compose.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ services:
121121
build:
122122
context: ../../apps/aifindr-evaluations-runner
123123
dockerfile: Dockerfile
124+
env_file:
125+
- ../../apps/aifindr-evaluations-runner/.env
124126
ports:
125127
- "8001:8001"
126128

0 commit comments

Comments
 (0)