fx

Author: galzilber

packages/traceloop-sdk/.flake8

@@ -11,4 +11,4 @@ exclude =
 max-line-length = 120
 per-file-ignores =
 	__init__.py:F401
-	traceloop/sdk/evaluators_generated/*.py:E501
+	traceloop/sdk/generated/**/*.py:E501

packages/traceloop-sdk/traceloop/sdk/evaluator/evaluator.py

@@ -12,7 +12,7 @@
 )
 from .stream_client import SSEClient
 from .config import EvaluatorDetails
-from ..evaluators_generated import get_request_model
+from ..generated.evaluators import get_request_model
 
 
 def _validate_evaluator_input(slug: str, input: Dict[str, str]) -> None:

packages/traceloop-sdk/traceloop/sdk/evaluator/evaluators_made_by_traceloop.py

@@ -1,12 +1,12 @@
 """
 Factory class for creating Traceloop evaluators with proper configuration.
 
-This module dynamically generates factory methods from the evaluators_generated registry.
+This module dynamically generates factory methods from the generated.evaluators registry.
 """
 
 from typing import Any, List
 
-from ..evaluators_generated import REQUEST_MODELS
+from ..generated.evaluators import REQUEST_MODELS
 from .config import EvaluatorDetails
 
 

packages/traceloop-sdk/traceloop/sdk/evaluator/model.py

@@ -55,7 +55,7 @@ def typed_result(self, model: Type[T]) -> T:
             An instance of the provided model class
 
         Example:
-            from traceloop.sdk.evaluators_generated import PIIDetectorResponse
+            from traceloop.sdk.generated.evaluators import PIIDetectorResponse
             result = await evaluator.run_experiment_evaluator(...)
             pii = result.typed_result(PIIDetectorResponse)
             print(pii.has_pii)  # IDE autocomplete works!

packages/traceloop-sdk/traceloop/sdk/generated/__init__.py

@@ -0,0 +1,16 @@
+# Generated code modules
+# DO NOT EDIT MANUALLY - Regenerate with scripts in /scripts/
+
+from .evaluators import (
+    REQUEST_MODELS,
+    RESPONSE_MODELS,
+    get_request_model,
+    get_response_model,
+)
+
+__all__ = [
+    "REQUEST_MODELS",
+    "RESPONSE_MODELS",
+    "get_request_model",
+    "get_response_model",
+]

…oop/sdk/evaluators_generated/__init__.py …oop/sdk/generated/evaluators/__init__.pypackages/traceloop-sdk/traceloop/sdk/evaluators_generated/__init__.py renamed to packages/traceloop-sdk/traceloop/sdk/generated/evaluators/__init__.py packages/traceloop-sdk/traceloop/sdk/evaluators_generated/__init__.py renamed to packages/traceloop-sdk/traceloop/sdk/generated/evaluators/__init__.py

@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  tmpcyjngtw9.json
+#   filename:  tmpvqz8m01b.json
 
 from __future__ import annotations
 
@@ -19,7 +19,7 @@ class AgentEfficiencyRequest(BaseModel):
 
 class AgentFlowQualityRequest(BaseModel):
     conditions: list[str] = Field(
-        ..., examples=[["no tools called", "agent completed task"]]
+        ..., examples=[['no tools called', 'agent completed task']]
     )
     threshold: float = Field(..., examples=[0.5])
     trajectory_completions: str = Field(
@@ -38,13 +38,13 @@ class AgentGoalAccuracyRequest(BaseModel):
     completion: str = Field(
         ...,
         examples=[
-            "I have booked your flight from New York to Los Angeles departing Monday at 9am."
+            'I have booked your flight from New York to Los Angeles departing Monday at 9am.'
         ],
     )
     question: str = Field(
-        ..., examples=["Book a flight from NYC to LA for next Monday"]
+        ..., examples=['Book a flight from NYC to LA for next Monday']
     )
-    reference: str = Field(..., examples=["Flight booked: NYC to LA, Monday departure"])
+    reference: str = Field(..., examples=['Flight booked: NYC to LA, Monday departure'])
 
 
 class AgentGoalCompletenessRequest(BaseModel):
@@ -72,42 +72,42 @@ class AgentToolErrorDetectorRequest(BaseModel):
 
 
 class AnswerCompletenessRequest(BaseModel):
-    completion: str = Field(..., examples=["Paris."])
-    context: str = Field(..., examples=["The capital of France is Paris."])
-    question: str = Field(..., examples=["What is the capital of France?"])
+    completion: str = Field(..., examples=['Paris.'])
+    context: str = Field(..., examples=['The capital of France is Paris.'])
+    question: str = Field(..., examples=['What is the capital of France?'])
 
 
 class AnswerCorrectnessRequest(BaseModel):
-    completion: str = Field(..., examples=["World War II ended in 1945."])
-    ground_truth: str = Field(..., examples=["1945"])
-    question: str = Field(..., examples=["What year did World War II end?"])
+    completion: str = Field(..., examples=['World War II ended in 1945.'])
+    ground_truth: str = Field(..., examples=['1945'])
+    question: str = Field(..., examples=['What year did World War II end?'])
 
 
 class AnswerRelevancyRequest(BaseModel):
-    answer: str = Field(..., examples=["The capital of France is Paris."])
-    question: str = Field(..., examples=["What is the capital of France?"])
+    answer: str = Field(..., examples=['The capital of France is Paris.'])
+    question: str = Field(..., examples=['What is the capital of France?'])
 
 
 class CharCountRatioRequest(BaseModel):
     denominator_text: str = Field(
-        ..., examples=["This is a longer text for comparison"]
+        ..., examples=['This is a longer text for comparison']
     )
-    numerator_text: str = Field(..., examples=["Short text"])
+    numerator_text: str = Field(..., examples=['Short text'])
 
 
 class CharCountRequest(BaseModel):
-    text: str = Field(..., examples=["Hello, world! This is a sample text."])
+    text: str = Field(..., examples=['Hello, world! This is a sample text.'])
 
 
 class ContextRelevanceRequest(BaseModel):
     context: str = Field(
         ...,
         examples=[
-            "Our store is open Monday to Friday from 9am to 6pm, and Saturday from 10am to 4pm. We are closed on Sundays."
+            'Our store is open Monday to Friday from 9am to 6pm, and Saturday from 10am to 4pm. We are closed on Sundays.'
         ],
     )
-    model: Optional[str] = Field(None, examples=["gpt-4o"])
-    query: str = Field(..., examples=["What are the business hours?"])
+    model: Optional[str] = Field(None, examples=['gpt-4o'])
+    query: str = Field(..., examples=['What are the business hours?'])
 
 
 class ConversationQualityRequest(BaseModel):
@@ -117,33 +117,33 @@ class ConversationQualityRequest(BaseModel):
             '["Hi! I\'d be happy to assist you today.", "We offer consulting, development, and support services."]'
         ],
     )
-    model: Optional[str] = Field(None, examples=["gpt-4o"])
+    model: Optional[str] = Field(None, examples=['gpt-4o'])
     prompts: str = Field(
         ..., examples=['["Hello, how can I help?", "What services do you offer?"]']
     )
 
 
 class FaithfulnessRequest(BaseModel):
     completion: str = Field(
-        ..., examples=["The Eiffel Tower is located in Paris and was built in 1889."]
+        ..., examples=['The Eiffel Tower is located in Paris and was built in 1889.']
     )
     context: str = Field(
         ...,
         examples=[
-            "The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France. It was constructed from 1887 to 1889."
+            'The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France. It was constructed from 1887 to 1889.'
         ],
     )
-    question: str = Field(..., examples=["When was the Eiffel Tower built?"])
+    question: str = Field(..., examples=['When was the Eiffel Tower built?'])
 
 
 class InstructionAdherenceRequest(BaseModel):
     instructions: str = Field(
-        ..., examples=["Respond in exactly 3 bullet points and use formal language."]
+        ..., examples=['Respond in exactly 3 bullet points and use formal language.']
     )
     response: str = Field(
         ...,
         examples=[
-            "- First point about the topic\n- Second relevant consideration\n- Final concluding thought"
+            '- First point about the topic\n- Second relevant consideration\n- Final concluding thought'
         ],
     )
 
@@ -155,7 +155,7 @@ class IntentChangeRequest(BaseModel):
             '["Sure, I can help with hotel booking", "No problem, let me search for flights"]'
         ],
     )
-    model: Optional[str] = Field(None, examples=["gpt-4o"])
+    model: Optional[str] = Field(None, examples=['gpt-4o'])
     prompts: str = Field(
         ...,
         examples=['["I want to book a hotel", "Actually, I need a flight instead"]'],
@@ -164,7 +164,7 @@ class IntentChangeRequest(BaseModel):
 
 class JSONValidatorRequest(BaseModel):
     enable_schema_validation: Optional[bool] = Field(None, examples=[True])
-    schema_string: Optional[str] = Field(None, examples=["{}"])
+    schema_string: Optional[str] = Field(None, examples=['{}'])
     text: str = Field(..., examples=['{"name": "John", "age": 30}'])
 
 
@@ -173,105 +173,105 @@ class PIIDetectorRequest(BaseModel):
     text: str = Field(
         ...,
         examples=[
-            "Please contact John Smith at john.smith@email.com or call 555-123-4567."
+            'Please contact John Smith at john.smith@email.com or call 555-123-4567.'
         ],
     )
 
 
 class PerplexityRequest(BaseModel):
-    logprobs: str = Field(..., examples=["[-2.3, -1.5, -0.8, -1.2, -0.5]"])
+    logprobs: str = Field(..., examples=['[-2.3, -1.5, -0.8, -1.2, -0.5]'])
 
 
 class PlaceholderRegexRequest(BaseModel):
     case_sensitive: Optional[bool] = Field(None, examples=[True])
     dot_include_nl: Optional[bool] = Field(None, examples=[True])
     multi_line: Optional[bool] = Field(None, examples=[True])
     placeholder_value: str = Field(
-        ..., examples=["[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}"]
+        ..., examples=['[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}']
     )
     should_match: Optional[bool] = Field(None, examples=[True])
-    text: str = Field(..., examples=["user@example.com"])
+    text: str = Field(..., examples=['user@example.com'])
 
 
 class ProfanityDetectorRequest(BaseModel):
-    text: str = Field(..., examples=["This is a clean and professional message."])
+    text: str = Field(..., examples=['This is a clean and professional message.'])
 
 
 class PromptInjectionRequest(BaseModel):
-    prompt: str = Field(..., examples=["What is the weather like today?"])
+    prompt: str = Field(..., examples=['What is the weather like today?'])
     threshold: Optional[float] = Field(None, examples=[0.5])
 
 
 class PromptPerplexityRequest(BaseModel):
-    prompt: str = Field(..., examples=["What is the capital of France?"])
+    prompt: str = Field(..., examples=['What is the capital of France?'])
 
 
 class RegexValidatorRequest(BaseModel):
     case_sensitive: Optional[bool] = Field(None, examples=[True])
     dot_include_nl: Optional[bool] = Field(None, examples=[True])
     multi_line: Optional[bool] = Field(None, examples=[True])
-    regex: Optional[str] = Field(None, examples=[".*"])
+    regex: Optional[str] = Field(None, examples=['.*'])
     should_match: Optional[bool] = Field(None, examples=[True])
-    text: str = Field(..., examples=["user@example.com"])
+    text: str = Field(..., examples=['user@example.com'])
 
 
 class SQLValidatorRequest(BaseModel):
-    text: str = Field(..., examples=["SELECT * FROM users WHERE id = 1;"])
+    text: str = Field(..., examples=['SELECT * FROM users WHERE id = 1;'])
 
 
 class SecretsDetectorRequest(BaseModel):
     text: str = Field(
-        ..., examples=["Here is some text without any API keys or passwords."]
+        ..., examples=['Here is some text without any API keys or passwords.']
     )
 
 
 class SemanticSimilarityRequest(BaseModel):
-    completion: str = Field(..., examples=["The cat sat on the mat."])
-    reference: str = Field(..., examples=["A feline was resting on the rug."])
+    completion: str = Field(..., examples=['The cat sat on the mat.'])
+    reference: str = Field(..., examples=['A feline was resting on the rug.'])
 
 
 class SexismDetectorRequest(BaseModel):
     text: str = Field(
         ...,
-        examples=["All team members should be treated equally regardless of gender."],
+        examples=['All team members should be treated equally regardless of gender.'],
     )
     threshold: Optional[float] = Field(None, examples=[0.5])
 
 
 class ToneDetectionRequest(BaseModel):
-    text: str = Field(..., examples=["The capital of France is Paris."])
+    text: str = Field(..., examples=['The capital of France is Paris.'])
 
 
 class TopicAdherenceRequest(BaseModel):
     completion: str = Field(
         ...,
         examples=[
-            "Machine learning is a subset of AI that enables systems to learn from data."
+            'Machine learning is a subset of AI that enables systems to learn from data.'
         ],
     )
-    question: str = Field(..., examples=["Tell me about machine learning"])
+    question: str = Field(..., examples=['Tell me about machine learning'])
     reference_topics: str = Field(
-        ..., examples=["artificial intelligence, data science, algorithms"]
+        ..., examples=['artificial intelligence, data science, algorithms']
     )
 
 
 class ToxicityDetectorRequest(BaseModel):
-    text: str = Field(..., examples=["Thank you for your help with this project."])
+    text: str = Field(..., examples=['Thank you for your help with this project.'])
     threshold: Optional[float] = Field(None, examples=[0.5])
 
 
 class UncertaintyDetectorRequest(BaseModel):
     prompt: str = Field(
-        ..., examples=["I am not sure, I think the capital of France is Paris."]
+        ..., examples=['I am not sure, I think the capital of France is Paris.']
     )
 
 
 class WordCountRatioRequest(BaseModel):
     denominator_text: str = Field(
-        ..., examples=["This is a longer input text for comparison"]
+        ..., examples=['This is a longer input text for comparison']
     )
-    numerator_text: str = Field(..., examples=["Short response"])
+    numerator_text: str = Field(..., examples=['Short response'])
 
 
 class WordCountRequest(BaseModel):
-    text: str = Field(..., examples=["This is a sample text with several words."])
+    text: str = Field(..., examples=['This is a sample text with several words.'])