diff --git a/pages/docs/observability/features/_meta.tsx b/pages/docs/observability/features/_meta.tsx index 23db8df38..219514d75 100644 --- a/pages/docs/observability/features/_meta.tsx +++ b/pages/docs/observability/features/_meta.tsx @@ -1,23 +1,30 @@ import { MenuSubSeparator } from "@/components/MenuSubSeparator"; export default { - "-- Core": { + "-- Essential": { type: "separator", title: Essential, }, + observations: {}, + traces: {}, sessions: {}, + + "-- Attributes": { + type: "separator", + title: Attributes, + }, users: {}, - environments: {}, - tags: {}, metadata: {}, - "trace-ids-and-distributed-tracing": {}, + tags: {}, + environments: {}, + "log-levels": {}, + "releases-and-versioning": {}, "-- Advanced": { type: "separator", title: Advanced, }, "user-feedback": {}, - "log-levels": {}, "*": { layout: "default", }, diff --git a/pages/docs/observability/features/environments.mdx b/pages/docs/observability/features/environments.mdx index 21b1fe512..fc4b0098f 100644 --- a/pages/docs/observability/features/environments.mdx +++ b/pages/docs/observability/features/environments.mdx @@ -15,24 +15,6 @@ You can configure the environment by setting the `LANGFUSE_TRACING_ENVIRONMENT` If both are specified, the initialization parameter takes precedence. If nothing is specified, the default environment is `default`. -## Data Model - -The `environment` attribute is available on all events in Langfuse: - -- Traces -- Observations (spans, events, generations) -- Scores -- Sessions - -See [Data Model](/docs/tracing-data-model) for more details. - -The environment must be a string that follows this regex pattern: `^(?!langfuse)[a-z0-9-_]+$` with at most 40 characters. - -This means: - -- Cannot start with "langfuse" -- Can only contain lowercase letters, numbers, hyphens, and underscores - ## Usage @@ -218,6 +200,24 @@ For guidance on how to structure, separate, and work with multiple environments 2. **Environment-Specific Analysis**: Use environments to analyze and compare metrics across different deployment stages. 3. **Testing**: Use separate environments for testing to avoid polluting production data. +## Data Model + +The `environment` attribute is available on all events in Langfuse: + +- Traces +- Observations (spans, events, generations) +- Scores +- Sessions + +See [Data Model](/docs/tracing-data-model) for more details. + +The environment must be a string that follows this regex pattern: `^(?!langfuse)[a-z0-9-_]+$` with at most 40 characters. + +This means: + +- Cannot start with "langfuse" +- Can only contain lowercase letters, numbers, hyphens, and underscores + ## GitHub Discussions import { GhDiscussionsPreview } from "@/components/gh-discussions/GhDiscussionsPreview"; diff --git a/pages/docs/observability/features/observations.mdx b/pages/docs/observability/features/observations.mdx new file mode 100644 index 000000000..28ab10853 --- /dev/null +++ b/pages/docs/observability/features/observations.mdx @@ -0,0 +1,480 @@ +--- +title: Observations +description: Observations are the building blocks of observability in Langfuse. +sidebarTitle: Observations +--- + +# Observations + +Observations are the building blocks of observability in Langfuse. They are used to track the individual steps of an application. + +Observations are either created automatically by one of our [integrations](/docs/integrations) or manually by you using the [Langfuse SDK](/docs/sdk/overview). + +## Observation Types + +Langfuse supports different observation types to provide more context to your observations and allow efficient filtering. + +import ObservationTypesList from "@/components-mdx/observation-types-list.mdx"; + +### Available Types + + + +### How to Use Observation Types + +The [integrations with agent frameworks](/docs/integrations) automatically set the observation types. For example, marking a method with `@tool` in langchain will automatically set the Langfuse observation type to `tool`. + +You can also manually set the observation types for your application within the Langfuse SDK. Set the `as_type` parameter (Python) or `asType` parameter (TypeScript) to the desired observation type when creating an observation. + + + + + + Observation types require Python SDK `version>=3.3.1`. + + +Using `@observe` decorator: + +```python /as_type="agent"/ /as_type="tool"/ +from langfuse import observe + +# Agent workflow +@observe(as_type="agent") +def run_agent_workflow(query): + # Agent reasoning and tool orchestration + return process_with_tools(query) + +# Tool calls +@observe(as_type="tool") +def call_weather_api(location): + # External API call + return weather_service.get_weather(location) +``` + +Calling the `start_as_current_observation` or `start_observation` method: + +```python /as_type="embedding"/ /as_type="chain"/ +from langfuse import get_client + +langfuse = get_client() + +# Start observation with specific type +with langfuse.start_as_current_observation( + as_type="embedding", + name="embedding-generation" +) as obs: + embeddings = model.encode(["text to embed"]) + obs.update(output=embeddings) + +# Start observation with specific type +transform_span = langfuse.start_observation( + as_type="chain", + name="transform-text" +) +transformed_text = transform_text(["text to transform"]) +transform_span.update(output=transformed_text) +``` + + + + + +Observation types are available since Typescript SDK `version>=4.0.0`. + + + + + +Use `startActiveObservation` with the `asType` option to specify observation types in context managers: + +```typescript /asType: "agent"/ /asType: "tool"/ /asType: "chain"/ /asType: "generation"/ /asType: "embedding"/ /asType: "retriever"/ /asType: "evaluator"/ /asType: "guardrail"/ +import { startActiveObservation } from "@langfuse/tracing"; + +// Agent workflow +await startActiveObservation( + "agent-workflow", + async (agentObservation) => { + agentObservation.update({ + input: { query: "What's the weather in Paris?" }, + metadata: { strategy: "tool-calling" } + }); + + // Agent reasoning and tool orchestration + const result = await processWithTools(query); + agentObservation.update({ output: result }); + }, + { asType: "agent" } +); + +// Tool call +await startActiveObservation( + "weather-api-call", + async (toolObservation) => { + toolObservation.update({ + input: { location: "Paris", units: "metric" }, + }); + + const weather = await weatherService.getWeather("Paris"); + toolObservation.update({ output: weather }); + }, + { asType: "tool" } +); + +// Chain operation +await startActiveObservation( + "retrieval-chain", + async (chainObservation) => { + chainObservation.update({ + input: { query: "AI safety principles" }, + }); + + const docs = await retrieveDocuments(query); + const context = await processDocuments(docs); + chainObservation.update({ output: { context, documentCount: docs.length } }); + }, + { asType: "chain" } +); +``` + +Examples for other observation types: + +```typescript /asType: "generation"/ /asType: "embedding"/ /asType: "retriever"/ +// LLM Generation +await startActiveObservation( + "llm-completion", + async (generationObservation) => { + generationObservation.update({ + input: [{ role: "user", content: "Explain quantum computing" }], + model: "gpt-4", + }); + + const completion = await openai.chat.completions.create({ + model: "gpt-4", + messages: [{ role: "user", content: "Explain quantum computing" }], + }); + + generationObservation.update({ + output: completion.choices[0].message.content, + usageDetails: { + input: completion.usage.prompt_tokens, + output: completion.usage.completion_tokens, + }, + }); + }, + { asType: "generation" } +); + +// Embedding generation +await startActiveObservation( + "text-embedding", + async (embeddingObservation) => { + const texts = ["Hello world", "How are you?"]; + embeddingObservation.update({ + input: texts, + model: "text-embedding-ada-002", + }); + + const embeddings = await openai.embeddings.create({ + model: "text-embedding-ada-002", + input: texts, + }); + + embeddingObservation.update({ + output: embeddings.data.map(e => e.embedding), + usageDetails: { input: embeddings.usage.prompt_tokens }, + }); + }, + { asType: "embedding" } +); + +// Document retrieval +await startActiveObservation( + "vector-search", + async (retrieverObservation) => { + retrieverObservation.update({ + input: { query: "machine learning", topK: 5 }, + }); + + const results = await vectorStore.similaritySearch(query, 5); + retrieverObservation.update({ + output: results, + metadata: { vectorStore: "pinecone", similarity: "cosine" }, + }); + }, + { asType: "retriever" } +); +``` + + + + +Use the `observe` wrapper with the `asType` option to automatically trace functions: + +```typescript /asType: "agent"/ /asType: "tool"/ /asType: "evaluator"/ +import { observe, updateActiveObservation } from "@langfuse/tracing"; + +// Agent function +const runAgentWorkflow = observe( + async (query: string) => { + updateActiveObservation({ + metadata: { strategy: "react", maxIterations: 5 } + }); + + // Agent logic here + return await processQuery(query); + }, + { + name: "agent-workflow", + asType: "agent" + } +); + +// Tool function +const callWeatherAPI = observe( + async (location: string) => { + updateActiveObservation({ + metadata: { provider: "openweather", version: "2.5" } + }); + + return await weatherService.getWeather(location); + }, + { + name: "weather-tool", + asType: "tool" + } +); + +// Evaluation function +const evaluateResponse = observe( + async (question: string, answer: string) => { + updateActiveObservation({ + metadata: { criteria: ["relevance", "accuracy", "completeness"] } + }); + + const score = await llmEvaluator.evaluate(question, answer); + return { score, feedback: "Response is accurate and complete" }; + }, + { + name: "response-evaluator", + asType: "evaluator" + } +); +``` + +More examples with different observation types: + +```typescript /asType: "generation"/ /asType: "chain"/ /asType: "guardrail"/ +// Generation wrapper +const generateCompletion = observe( + async (messages: any[], model: string = "gpt-4") => { + updateActiveObservation({ + model, + metadata: { temperature: 0.7, maxTokens: 1000 } + }, { asType: "generation" }); + + const completion = await openai.chat.completions.create({ + model, + messages, + temperature: 0.7, + max_tokens: 1000, + }); + + updateActiveObservation({ + usageDetails: { + input: completion.usage.prompt_tokens, + output: completion.usage.completion_tokens, + } + }, { asType: "generation" }); + + return completion.choices[0].message.content; + }, + { + name: "llm-completion", + asType: "generation" + } +); + +// Chain wrapper +const processDocumentChain = observe( + async (documents: string[]) => { + updateActiveObservation({ + metadata: { documentCount: documents.length } + }); + + const summaries = await Promise.all( + documents.map(doc => summarizeDocument(doc)) + ); + + return await combineAndRank(summaries); + }, + { + name: "document-processing-chain", + asType: "chain" + } +); + +// Guardrail wrapper +const contentModerationCheck = observe( + async (content: string) => { + updateActiveObservation({ + metadata: { provider: "openai-moderation", version: "stable" } + }); + + const moderation = await openai.moderations.create({ + input: content, + }); + + const flagged = moderation.results[0].flagged; + updateActiveObservation({ + output: { flagged, categories: moderation.results[0].categories } + }); + + if (flagged) { + throw new Error("Content violates usage policies"); + } + + return { safe: true, content }; + }, + { + name: "content-guardrail", + asType: "guardrail" + } +); +``` + + + + +Use `startObservation` with the `asType` option for manual observation management: + +```typescript /asType: "agent"/ /asType: "tool"/ /asType: "generation"/ +import { startObservation } from "@langfuse/tracing"; + +// Agent observation +const agentSpan = startObservation( + "multi-step-agent", + { + input: { task: "Book a restaurant reservation" }, + metadata: { agentType: "planning", tools: ["search", "booking"] } + }, + { asType: "agent" } +) + +// Nested tool calls within the agent +const searchTool = agentSpan.startObservation( + "restaurant-search", + { + input: { location: "New York", cuisine: "Italian", date: "2024-01-15" } + }, + { asType: "tool" } +); + +searchTool.update({ + output: { restaurants: ["Mario's", "Luigi's"], count: 2 } +}); +searchTool.end(); + +const bookingTool = agentSpan.startObservation( + "make-reservation", + { + input: { restaurant: "Mario's", time: "7:00 PM", party: 4 } + }, + { asType: "tool" } +); + +bookingTool.update({ + output: { confirmed: true, reservationId: "RES123" } +}); +bookingTool.end(); + +agentSpan.update({ + output: { success: true, reservationId: "RES123" } +}); +agentSpan.end(); +``` + +Examples with other observation types: + +```typescript /asType: "embedding"/ /asType: "retriever"/ /asType: "evaluator"/ +// Embedding observation +const embeddingObs = startObservation( + "document-embedding", + { + input: ["Document 1 content", "Document 2 content"], + model: "text-embedding-ada-002" + }, + { asType: "embedding" } +); + +const embeddings = await generateEmbeddings(documents); +embeddingObs.update({ + output: embeddings, + usageDetails: { input: 150 } +}); +embeddingObs.end(); + +// Retriever observation +const retrieverObs = startObservation( + "semantic-search", + { + input: { query: "What is machine learning?", topK: 10 }, + metadata: { index: "knowledge-base", similarity: "cosine" } + }, + { asType: "retriever" } +); + +const searchResults = await vectorDB.search(query, 10); +retrieverObs.update({ + output: { documents: searchResults, scores: searchResults.map(r => r.score) } +}); +retrieverObs.end(); + +// Evaluator observation +const evalObs = startObservation( + "hallucination-check", + { + input: { + context: "The capital of France is Paris.", + response: "The capital of France is London." + }, + metadata: { evaluator: "llm-judge", model: "gpt-4" } + }, + { asType: "evaluator" } +); + +const evaluation = await checkHallucination(context, response); +evalObs.update({ + output: { + score: 0.1, + reasoning: "Response contradicts the provided context", + verdict: "hallucination_detected" + } +}); +evalObs.end(); + +// Guardrail observation +const guardrailObs = startObservation( + "safety-filter", + { + input: { userMessage: "How to make explosives?" }, + metadata: { policy: "content-safety-v2" } + }, + { asType: "guardrail" } +); + +const safetyCheck = await contentFilter.check(userMessage); +guardrailObs.update({ + output: { + blocked: true, + reason: "harmful_content", + category: "dangerous_instructions" + } +}); +guardrailObs.end(); +``` + + + + + + diff --git a/pages/docs/observability/features/sessions.mdx b/pages/docs/observability/features/sessions.mdx index 5db1eb96e..927bc3fcb 100644 --- a/pages/docs/observability/features/sessions.mdx +++ b/pages/docs/observability/features/sessions.mdx @@ -8,7 +8,7 @@ import { PropagationRestrictionsCallout } from "@/components/PropagationRestrict # Sessions -Many interactions with LLM applications span multiple traces and observations. `Sessions` in Langfuse are a special way to group these observations across traces together and see a simple **session replay** of the entire interaction. Get started by propagating the `sessionId` attribute across observations. +Many interactions with LLM applications span multiple [traces](/docs/observability/features/traces) and [observations](/docs/observability/features/observations). `Sessions` in Langfuse are a special way to group these observations across traces together and see a simple **session replay** of the entire interaction. Get started by propagating the `sessionId` attribute across observations. ```mermaid graph LR diff --git a/pages/docs/observability/features/trace-ids-and-distributed-tracing.mdx b/pages/docs/observability/features/traces.mdx similarity index 83% rename from pages/docs/observability/features/trace-ids-and-distributed-tracing.mdx rename to pages/docs/observability/features/traces.mdx index 6392c5c91..9cec4d480 100644 --- a/pages/docs/observability/features/trace-ids-and-distributed-tracing.mdx +++ b/pages/docs/observability/features/traces.mdx @@ -1,10 +1,18 @@ --- title: Trace IDs & Distributed Tracing description: Bring your own trace IDs for distributed tracing and linking traces across services. -sidebarTitle: Trace IDs & Distributed Tracing +sidebarTitle: Traces --- -# Trace IDs & Distributed Tracing +# Traces + +A trace groups together multiple [observations](/docs/observability/features/observations) that are part of the same request or operation. + +For example, when a user asks a question to a chatbot, that interaction, from the user’s question to the bot’s response, is captured as one trace. It contains the overall input and output of the function, as well as metadata about the request ( i.e. user, session, tags, etc.). + +Most integrations will automatically create a trace for you. However, you can also create a trace manually using the Langfuse SDK. + +## Trace IDs & Distributed Tracing Langfuse allows you to bring your own trace IDs (e.g., messageId, traceId, correlationId) for @@ -46,6 +54,13 @@ The Python SDK uses W3C Trace Context IDs by default, which are: - 32-character lowercase hexadecimal string for trace IDs - 16-character lowercase hexadecimal string for observation (span) IDs +You can inspect and reuse these IDs programmatically: + +- `langfuse.get_current_trace_id()` returns the trace ID of the active observation. +- `langfuse.get_current_observation_id()` returns the observation (span) ID of the active observation. +- `span.trace_id` and `span.id` expose the IDs on any `LangfuseSpan`/`LangfuseGeneration` returned from `start_as_current_observation`. +- `Langfuse.create_trace_id(seed: Optional[str] = None)` is a static helper to deterministically pre-generate trace IDs before a trace exists (e.g., to correlate an external request ID). + ### Using the Decorator ```python @@ -114,7 +129,7 @@ with langfuse.start_as_current_observation( # Your code here ``` -### Accessing Current Trace ID +### Accessing Current Trace & Observation IDs ```python from langfuse import get_client @@ -122,12 +137,37 @@ from langfuse import get_client langfuse = get_client() with langfuse.start_as_current_observation(as_type="span", name="outer-operation") as span: - # Access the trace ID of the current span + # Access the IDs of the current span current_trace_id = langfuse.get_current_trace_id() current_span_id = langfuse.get_current_observation_id() - print(f"Current trace ID: {current_trace_id}") + print(f"Trace ID from helper: {current_trace_id}") + print(f"Observation ID from helper: {current_span_id}") + print(f"Trace ID from object: {span.trace_id}") + print(f"Observation ID from object: {span.id}") + +``` + +### Pre-generating Trace IDs + +Use the static helper when you need a trace ID before creating any spans (for example, to pass IDs between services or to link later scores to a trace that does not yet exist): + +```python +from langfuse import get_client, Langfuse + +langfuse = get_client() + +external_request_id = "req_12345" +deterministic_trace_id = Langfuse.create_trace_id(seed=external_request_id) +print(f"Deterministic Trace ID for {external_request_id}: {deterministic_trace_id}") +# Later, start a span that adopts that trace ID +with langfuse.start_as_current_observation( + as_type="span", + name="process-request", + trace_context={"trace_id": deterministic_trace_id} +) as span: + # Your code here ```