Skip to content

Commit 52ddf46

Browse files
neuromechanistneuromechanist
authored
Merge pull request #80 from Annotation-Garden/develop
Release 0.6.4a1: LiteLLM prompt caching integration
2 parents 7860c96 + 0922cd4 commit 52ddf46

15 files changed

+1671
-68
lines changed

‎deploy/auto-update-dev.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,16 @@ if [ "$RUNNING_ID" != "$NEW_ID" ]; then
3737
FEEDBACK_DIR="/var/lib/hedit/${CONTAINER_NAME}/feedback"
3838
mkdir -p "${FEEDBACK_DIR}/unprocessed" "${FEEDBACK_DIR}/processed" 2>/dev/null || true
3939

40+
# Create persistent telemetry directory
41+
TELEMETRY_DIR="/var/lib/hedit/${CONTAINER_NAME}/telemetry"
42+
mkdir -p "${TELEMETRY_DIR}" 2>/dev/null || true
43+
4044
docker run -d \
4145
--name "$CONTAINER_NAME" \
4246
--restart unless-stopped \
4347
-p "127.0.0.1:${HOST_PORT}:38427" \
4448
-v "${FEEDBACK_DIR}:/app/feedback" \
49+
-v "${TELEMETRY_DIR}:/app/telemetry" \
4550
$ENV_ARGS \
4651
"$REGISTRY_IMAGE" > /dev/null
4752

‎deploy/auto-update.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,11 @@ deploy_update() {
163163
mkdir -p "${FEEDBACK_DIR}/unprocessed" "${FEEDBACK_DIR}/processed" 2>/dev/null || true
164164
log "Feedback directory: ${FEEDBACK_DIR}"
165165

166+
# Create persistent telemetry directory
167+
TELEMETRY_DIR="/var/lib/hedit/${CONTAINER_NAME}/telemetry"
168+
mkdir -p "${TELEMETRY_DIR}" 2>/dev/null || true
169+
log "Telemetry directory: ${TELEMETRY_DIR}"
170+
166171
# Run the new container using the pulled image
167172
log "Starting new container on port ${HOST_PORT}..."
168173
docker run -d \
@@ -172,6 +177,7 @@ deploy_update() {
172177
${ENV_ARGS} \
173178
-v /var/log/hedit:/var/log/hedit \
174179
-v "${FEEDBACK_DIR}:/app/feedback" \
180+
-v "${TELEMETRY_DIR}:/app/telemetry" \
175181
"$REGISTRY_IMAGE"
176182

177183
if [ $? -eq 0 ]; then

‎docker-compose.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ services:
3434
- ollama
3535
volumes:
3636
- feedback_data:/app/feedback
37+
- telemetry_data:/app/telemetry
3738
environment:
3839
# LLM Provider Configuration
3940
- LLM_PROVIDER=${LLM_PROVIDER:-ollama}
@@ -73,3 +74,5 @@ volumes:
7374
driver: local
7475
feedback_data:
7576
driver: local
77+
telemetry_data:
78+
driver: local

‎frontend/index.html‎

Lines changed: 94 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -927,6 +927,61 @@
927927
visibility: visible;
928928
opacity: 1;
929929
}
930+
931+
/* Telemetry Notice - fixed at absolute bottom */
932+
.telemetry-notice {
933+
display: flex;
934+
align-items: baseline;
935+
justify-content: center;
936+
gap: 8px;
937+
padding: 4px 16px;
938+
font-size: 11px;
939+
color: var(--text-muted);
940+
border-top: 1px solid var(--border-color);
941+
background: var(--bg-tertiary);
942+
transition: all 0.3s ease;
943+
margin: 0;
944+
}
945+
946+
.telemetry-notice a {
947+
color: var(--text-secondary);
948+
text-decoration: underline;
949+
}
950+
951+
.telemetry-notice a:hover {
952+
color: #184A3D;
953+
}
954+
955+
html.dark .telemetry-notice a:hover {
956+
color: #9bc2b6;
957+
}
958+
959+
.telemetry-toggle {
960+
display: inline-flex;
961+
align-items: baseline;
962+
gap: 4px;
963+
cursor: pointer;
964+
user-select: none;
965+
}
966+
967+
.telemetry-toggle input[type="checkbox"] {
968+
width: 12px;
969+
height: 12px;
970+
margin: 0;
971+
cursor: pointer;
972+
accent-color: #184A3D;
973+
position: relative;
974+
top: 1px;
975+
}
976+
977+
html.dark .telemetry-toggle input[type="checkbox"] {
978+
accent-color: #9bc2b6;
979+
}
980+
981+
.telemetry-toggle-label {
982+
font-size: 11px;
983+
color: var(--text-muted);
984+
}
930985
</style>
931986
</head>
932987
<body>
@@ -1118,7 +1173,16 @@ <h3>Status</h3>
11181173
&copy; 2025, <a href="https://neuromechanist.github.io" target="_blank" rel="noopener noreferrer">neuromechanist</a> and the HED Working Group
11191174
</div>
11201175
</div>
1121-
</div>
1176+
</div>
1177+
1178+
<!-- Telemetry Notice - outside container for absolute bottom positioning -->
1179+
<div class="telemetry-notice">
1180+
<span>Annotations may be recorded to improve the service.</span>
1181+
<label class="telemetry-toggle" title="Toggle telemetry collection">
1182+
<input type="checkbox" id="telemetryOptIn" checked onchange="toggleTelemetry(this.checked)">
1183+
<span class="telemetry-toggle-label">Allow</span>
1184+
</label>
1185+
<a href="https://docs.annotation.garden/projects/hedit/telemetry" target="_blank" rel="noopener noreferrer" title="Learn more about telemetry">Learn more</a>
11221186
</div>
11231187

11241188
<script src="config.js"></script>
@@ -1166,6 +1230,31 @@ <h3>Status</h3>
11661230
// Initialize theme on page load
11671231
initTheme();
11681232

1233+
// Telemetry Toggle Functionality
1234+
let telemetryEnabled = true;
1235+
1236+
function initTelemetry() {
1237+
const saved = localStorage.getItem('telemetryEnabled');
1238+
// Default to true (opt-out model)
1239+
telemetryEnabled = saved === null ? true : saved === 'true';
1240+
const checkbox = document.getElementById('telemetryOptIn');
1241+
if (checkbox) {
1242+
checkbox.checked = telemetryEnabled;
1243+
}
1244+
}
1245+
1246+
function toggleTelemetry(enabled) {
1247+
telemetryEnabled = enabled;
1248+
localStorage.setItem('telemetryEnabled', enabled.toString());
1249+
}
1250+
1251+
function isTelemetryEnabled() {
1252+
return telemetryEnabled;
1253+
}
1254+
1255+
// Initialize telemetry on page load
1256+
initTelemetry();
1257+
11691258
// Backend API URL - Automatically detect based on environment
11701259
// For production: Set BACKEND_URL in your environment or use Cloudflare Tunnel URL
11711260
// For local development: Uses localhost:38427
@@ -1401,7 +1490,8 @@ <h3>Status</h3>
14011490
image: uploadedImageBase64,
14021491
schema_version: schema,
14031492
max_validation_attempts: maxAttempts,
1404-
run_assessment: runAssessment
1493+
run_assessment: runAssessment,
1494+
telemetry_enabled: isTelemetryEnabled()
14051495
};
14061496

14071497
if (visionPrompt) {
@@ -1516,7 +1606,8 @@ <h3>Generated Image Description</h3>
15161606
description: description,
15171607
schema_version: schema,
15181608
max_validation_attempts: maxAttempts,
1519-
run_assessment: runAssessment
1609+
run_assessment: runAssessment,
1610+
telemetry_enabled: isTelemetryEnabled()
15201611
};
15211612

15221613
// Include Turnstile token if available

‎pyproject.toml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
44

55
[project]
66
name = "hedit"
7-
version = "0.6.4a0"
7+
version = "0.6.4a1"
88
description = "Multi-agent system for HED annotation generation and validation"
99
readme = "PKG_README.md"
1010
requires-python = ">=3.12"
@@ -54,6 +54,8 @@ standalone = [
5454
"langchain-community>=0.3.0",
5555
"langchain-core>=0.3.0",
5656
"langchain-openai>=0.3.0",
57+
"litellm>=1.50.0", # LLM proxy with native prompt caching support
58+
"langchain-litellm>=0.2.0", # LangChain integration for LiteLLM
5759
"hedtools>=0.5.0",
5860
"lxml>=5.3.0",
5961
"beautifulsoup4>=4.12.3",

‎src/api/main.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
ValidationResponse,
3434
)
3535
from src.api.security import api_key_auth, audit_logger
36+
from src.telemetry import LocalFileStorage, TelemetryCollector, TelemetryEvent
3637
from src.utils.openrouter_llm import create_openrouter_llm, get_model_name
3738
from src.utils.schema_loader import HedSchemaLoader
3839
from src.validation.hed_validator import HedPythonValidator
@@ -45,6 +46,9 @@
4546
vision_agent: VisionAgent | None = None
4647
schema_loader: HedSchemaLoader | None = None
4748

49+
# Telemetry collector (initialized in lifespan)
50+
telemetry_collector: TelemetryCollector | None = None
51+
4852
# Cache for BYOK configuration
4953
_byok_config: dict = {}
5054

@@ -389,6 +393,18 @@ def get_default_path(docker_path: str, local_path: str) -> str:
389393
else:
390394
print("Vision model not available (only supported with OpenRouter)")
391395

396+
# Initialize telemetry collector
397+
global telemetry_collector
398+
# Use /app/telemetry in Docker, otherwise use local .hedit/telemetry
399+
default_telemetry_dir = "/app/telemetry" if Path("/app").exists() else ".hedit/telemetry"
400+
telemetry_dir = os.getenv("TELEMETRY_DIR", default_telemetry_dir)
401+
telemetry_storage = LocalFileStorage(storage_dir=telemetry_dir)
402+
telemetry_collector = TelemetryCollector(
403+
storage=telemetry_storage,
404+
enabled=True, # Can be configured via env var if needed
405+
)
406+
print(f"Telemetry collector initialized (storage: {telemetry_dir})")
407+
392408
yield
393409

394410
# Shutdown
@@ -573,20 +589,50 @@ async def annotate(
573589
# LangGraph default is 25, increase to 100 for complex workflows
574590
config = {"recursion_limit": 100}
575591

592+
start_time = time.time()
576593
final_state = await active_workflow.run(
577594
input_description=request.description,
578595
schema_version=request.schema_version,
579596
max_validation_attempts=request.max_validation_attempts,
580597
run_assessment=request.run_assessment,
581598
config=config,
582599
)
600+
latency_ms = int((time.time() - start_time) * 1000)
583601

584602
# Determine overall status
585603
# IMPORTANT: Ensure is_valid is only True when there are NO validation errors
586604
# This is a safeguard to prevent inconsistencies in the workflow
587605
is_valid = final_state["is_valid"] and len(final_state["validation_errors"]) == 0
588606
status = "success" if is_valid else "failed"
589607

608+
# Collect telemetry if enabled
609+
if request.telemetry_enabled and telemetry_collector:
610+
# Get model info from request body, BYOK headers, or server config
611+
model_name = (
612+
request.model
613+
or req.headers.get("x-openrouter-model")
614+
or os.getenv("ANNOTATION_MODEL", "openai/gpt-oss-120b")
615+
)
616+
temperature = (
617+
request.temperature
618+
or float(req.headers.get("x-openrouter-temperature", 0))
619+
or _byok_config.get("temperature", 0.1)
620+
)
621+
622+
event = TelemetryEvent.create(
623+
description=request.description,
624+
schema_version=request.schema_version,
625+
hed_string=final_state["current_annotation"],
626+
iterations=final_state["validation_attempts"],
627+
validation_errors=final_state["validation_errors"],
628+
model=model_name,
629+
provider=request.provider or req.headers.get("x-openrouter-provider"),
630+
temperature=temperature,
631+
latency_ms=latency_ms,
632+
source="api",
633+
)
634+
await telemetry_collector.collect(event)
635+
590636
return AnnotationResponse(
591637
annotation=final_state["current_annotation"],
592638
is_valid=is_valid,
@@ -682,6 +728,8 @@ async def annotate_from_image(
682728
active_vision_agent = vision_agent
683729

684730
try:
731+
start_time = time.time()
732+
685733
# Step 1: Generate image description using vision model
686734
vision_result = await active_vision_agent.describe_image(
687735
image_data=request.image,
@@ -701,11 +749,40 @@ async def annotate_from_image(
701749
run_assessment=request.run_assessment,
702750
config=config,
703751
)
752+
latency_ms = int((time.time() - start_time) * 1000)
704753

705754
# Determine overall status
706755
is_valid = final_state["is_valid"] and len(final_state["validation_errors"]) == 0
707756
status = "success" if is_valid else "failed"
708757

758+
# Collect telemetry if enabled
759+
if request.telemetry_enabled and telemetry_collector:
760+
# Get model info from request body, BYOK headers, or server config
761+
model_name = (
762+
request.model
763+
or req.headers.get("x-openrouter-model")
764+
or os.getenv("ANNOTATION_MODEL", "openai/gpt-oss-120b")
765+
)
766+
temperature = (
767+
request.temperature
768+
or float(req.headers.get("x-openrouter-temperature", 0))
769+
or _byok_config.get("temperature", 0.1)
770+
)
771+
772+
event = TelemetryEvent.create(
773+
description=image_description, # Use generated image description
774+
schema_version=request.schema_version,
775+
hed_string=final_state["current_annotation"],
776+
iterations=final_state["validation_attempts"],
777+
validation_errors=final_state["validation_errors"],
778+
model=model_name,
779+
provider=request.provider or req.headers.get("x-openrouter-provider"),
780+
temperature=temperature,
781+
latency_ms=latency_ms,
782+
source="api-image", # Distinguish from text-based annotation
783+
)
784+
await telemetry_collector.collect(event)
785+
709786
return ImageAnnotationResponse(
710787
image_description=image_description,
711788
annotation=final_state["current_annotation"],

‎src/api/models.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ class AnnotationRequest(BaseModel):
5555
le=1.0,
5656
examples=[0.1, 0.3, 0.7],
5757
)
58+
telemetry_enabled: bool = Field(
59+
default=True,
60+
description="Allow telemetry collection for this request",
61+
)
5862

5963

6064
class AnnotationResponse(BaseModel):
@@ -183,6 +187,10 @@ class ImageAnnotationRequest(BaseModel):
183187
le=1.0,
184188
examples=[0.1, 0.3, 0.7],
185189
)
190+
telemetry_enabled: bool = Field(
191+
default=True,
192+
description="Allow telemetry collection for this request",
193+
)
186194

187195

188196
class ImageAnnotationResponse(BaseModel):

0 commit comments

Comments
 (0)