From 024789e259e3a66a5bc03417339d4754997d549d Mon Sep 17 00:00:00 2001 From: Kathryn May Date: Wed, 13 Aug 2025 13:56:22 -0400 Subject: [PATCH 1/4] docs: consolidate manage datasets dropdown content to one page --- src/docs.json | 408 ++++++++++++++++++ src/langsmith/dataset-subset.mdx | 93 ---- src/langsmith/dataset-version.mdx | 58 --- src/langsmith/evaluation-overview.mdx | 2 +- .../export-filtered-traces-to-dataset.mdx | 22 - .../manage-datasets-programmatically.mdx | 2 +- src/langsmith/manage-datasets.mdx | 258 +++++++++++ src/langsmith/manage-organization-by-api.mdx | 2 +- src/langsmith/share-dataset.mdx | 30 -- src/langsmith/version-datasets.mdx | 47 -- 10 files changed, 669 insertions(+), 253 deletions(-) delete mode 100644 src/langsmith/dataset-subset.mdx delete mode 100644 src/langsmith/dataset-version.mdx delete mode 100644 src/langsmith/export-filtered-traces-to-dataset.mdx create mode 100644 src/langsmith/manage-datasets.mdx delete mode 100644 src/langsmith/share-dataset.mdx delete mode 100644 src/langsmith/version-datasets.mdx diff --git a/src/docs.json b/src/docs.json index 80169a115..9076c90e6 100644 --- a/src/docs.json +++ b/src/docs.json @@ -316,6 +316,414 @@ } ] }, + { + "dropdown": "LangSmith", + "icon": "/images/brand/langsmith-pill.svg", + "description": "Platform for LLM observability and evaluation", + "tabs": [ + { + "tab": "Get started", + "pages": [ + "langsmith/home", + + { + "group": "Start tracing", + "pages": [ + "langsmith/observability-overview", + "langsmith/observability-quickstart" + ] + }, + { + "group": "Evaluate your application", + "pages": [ + "langsmith/evaluation-overview", + "langsmith/evaluation-quickstart" + ] + }, + { + "group": "Test your prompts", + "pages": [ + "langsmith/prompt-engineering-overview", + "langsmith/prompt-engineering-quickstart-ui", + "langsmith/prompt-engineering-quickstart-sdk" + ] + }, + { + "group": "API & SDKs", + "pages": [ + "langsmith/api-ref", + "langsmith/python-sdk", + "langsmith/js-ts-sdk" + ] + }, + { + "group": "Pricing", + "pages": [ + "langsmith/pricing-plans", + "langsmith/pricing-faq" + ] + } + ] + }, + { + "tab": "Observability", + "pages": [ + "langsmith/observability-tab", + { + "group": "Set up tracing", + "pages": [ + { + "group": "Manual", + "pages": [ + "langsmith/annotate-code", + "langsmith/trace-with-api" + ] + }, + { + "group": "Integrations", + "pages": [ + "langsmith/trace-with-langchain", + "langsmith/trace-with-langgraph", + "langsmith/trace-anthropic", + "langsmith/trace-openai", + "langsmith/trace-with-instructor", + "langsmith/trace-with-vercel-ai-sdk", + "langsmith/trace-with-opentelemetry", + "langsmith/trace-with-openai-agents-sdk", + "langsmith/trace-claude-code" + ] + }, + { + "group": "Configuration", + "pages": [ + "langsmith/log-traces-to-project", + "langsmith/trace-without-env-vars", + "langsmith/threads", + "langsmith/sample-traces", + "langsmith/add-metadata-tags", + "langsmith/distributed-tracing", + "langsmith/access-current-span", + "langsmith/log-llm-trace", + "langsmith/calculate-token-based-costs", + "langsmith/log-multimodal-traces", + "langsmith/log-retriever-trace", + "langsmith/mask-inputs-outputs", + "langsmith/trace-generator-functions", + "langsmith/serverless-environments", + "langsmith/nest-traces", + "langsmith/upload-files-with-traces", + "langsmith/output-detailed-logs", + "langsmith/troubleshooting-variable-caching", + "langsmith/collector-proxy" + ] + } + ] + }, + { + "group": "View traces", + "pages": [ + "langsmith/filter-traces-in-application", + "langsmith/export-traces", + "langsmith/data-export", + "langsmith/share-trace", + "langsmith/compare-traces", + "langsmith/langgraph-platform-logs" + ] + }, + { + "group": "Monitoring", + "pages": [ + "langsmith/dashboards", + "langsmith/alerts", + "langsmith/alerts-pagerduty", + "langsmith/alerts-webhook" + ] + }, + { + "group": "Automations", + "pages": [ + "langsmith/rules", + "langsmith/webhooks", + "langsmith/online-evaluations" + ] + }, + { + "group": "Human Feedback", + "pages": [ + "langsmith/attach-user-feedback" + ] + }, + { + "group": "Trace a RAG application", + "pages": [ + "langsmith/observability-llm-tutorial" + ] + } + ] + }, + { + "tab": "Evaluation", + "pages": [ + "langsmith/evaluation-tab", + { + "group": "Datasets", + "pages": [ + { + "group": "Create a dataset", + "pages": [ + "langsmith/manage-datasets-in-application", + "langsmith/manage-datasets-programmatically" + ] + }, + "langsmith/manage-datasets" + ] + }, + { + "group": "Evaluations", + "pages": [ + { + "group": "Run an evaluation", + "pages": [ + "langsmith/evaluate-llm-application", + "langsmith/run-evaluation-from-prompt-playground", + { + "group": "Prebuilt evaluators", + "pages": [ + "langsmith/prebuilt-evaluators" + ] + } + ] + }, + { + "group": "Evaluation methods", + "pages": [ + "langsmith/evaluation-async", + "langsmith/pytest", + "langsmith/vitest-jest", + "langsmith/run-evals-api-only" + ] + }, + { + "group": "Evaluator types", + "pages": [ + "langsmith/code-evaluator", + "langsmith/llm-as-judge", + "langsmith/summary", + "langsmith/evaluate-pairwise" + ] + }, + { + "group": "Evaluation techniques", + "pages": [ + "langsmith/define-target-function", + "langsmith/evaluate-on-intermediate-steps", + "langsmith/multiple-scores", + "langsmith/metric-type", + "langsmith/repetition", + "langsmith/rate-limiting", + "langsmith/local", + "langsmith/langchain-runnable", + "langsmith/evaluate-graph", + "langsmith/evaluate-existing-experiment", + "langsmith/evaluate-with-attachments", + "langsmith/multi-turn-simulation" + ] + }, + { + "group": "Improve evaluators", + "pages": [ + "langsmith/improve-judge-evaluator-feedback", + "langsmith/create-few-shot-evaluators", + "langsmith/index-datasets-for-dynamic-few-shot-example-selection" + ] + } + ] + }, + { + "group": "Analyze experiment results", + "pages": [ + "langsmith/analyze-single-experiment", + "langsmith/compare-experiment-results", + "langsmith/filter-experiments-ui", + "langsmith/fetch-perf-metrics-experiment", + "langsmith/upload-existing-experiments", + "langsmith/download-experiment-results-as-csv", + "langsmith/renaming-experiment", + "langsmith/bind-evaluator-to-dataset" + ] + }, + { + "group": "Annotation & human feedback", + "pages": [ + "langsmith/annotation-queues", + "langsmith/set-up-feedback-criteria", + "langsmith/annotate-traces-inline", + "langsmith/audit-evaluator-scores" + ] + }, + { + "group": "Tutorials", + "pages": [ + "langsmith/evaluate-chatbot-tutorial", + "langsmith/evaluate-rag-tutorial", + "langsmith/test-react-agent-pytest", + "langsmith/evaluate-complex-agent", + "langsmith/run-backtests-new-agent" + ] + } + ] + }, + { + "tab": "Prompt engineering", + "pages": [ + "langsmith/prompt-engineering-tab", + { + "group": "Create and update prompts", + "pages": [ + "langsmith/create-a-prompt", + "langsmith/manage-prompts-programmatically", + "langsmith/managing-model-configurations", + "langsmith/use-tools", + "langsmith/multimodal-content", + "langsmith/prompt-canvas", + { + "group": "Connect to models", + "pages": [ + "langsmith/custom-openai-compliant-model", + "langsmith/custom-endpoint" + ] + } + ] + }, + { + "group": "Manage prompts", + "pages": [ + "langsmith/prompt-tags", + "langsmith/trigger-webhook", + "langsmith/langchain-hub" + ] + }, + { + "group": "Tutorials", + "pages": [ + "langsmith/optimize-classifier", + "langsmith/prompt-commit", + "langsmith/multiple-messages" + ] + } + ] + }, + { + "tab": "Self-hosting", + "pages": [ + "langsmith/architectural-overview", + { + "group": "Setup", + "pages": [ + "langsmith/kubernetes", + "langsmith/docker", + "langsmith/self-host-usage", + "langsmith/self-host-upgrades", + "langsmith/self-host-egress", + "langsmith/self-host-organization-charts", + "langsmith/langsmith-managed-clickhouse" + ] + }, + { + "group": "Configuration", + "pages": [ + "langsmith/self-host-scale", + "langsmith/self-host-ttl", + "langsmith/self-host-ingress", + "langsmith/self-host-mirroring-images", + "langsmith/self-host-playground-environment-settings" + ] + }, + { + "group": "Authentication & access control", + "pages": [ + "langsmith/self-host-basic-auth", + "langsmith/self-host-sso", + "langsmith/self-host-user-management", + "langsmith/self-host-custom-tls-certificates", + "langsmith/self-host-using-an-existing-secret" + ] + }, + { + "group": "Connect external services", + "pages": [ + "langsmith/self-host-blob-storage", + "langsmith/self-host-external-clickhouse", + "langsmith/self-host-external-postgres", + "langsmith/self-host-external-redis" + ] + }, + { + "group": "Scripts", + "pages": [ + "langsmith/script-delete-a-workspace", + "langsmith/script-delete-an-organization", + "langsmith/script-delete-traces", + "langsmith/script-generate-clickhouse-stats", + "langsmith/script-generate-query-stats", + "langsmith/script-running-pg-support-queries", + "langsmith/script-running-ch-support-queries" + ] + }, + { + "group": "Observability", + "pages": [ + "langsmith/export-backend", + "langsmith/langsmith-collector", + "langsmith/observability-stack" + ] + } + ] + }, + { + "tab": "Administration", + "pages": [ + { + "group": "Setup", + "pages": [ + "langsmith/administration-overview", + "langsmith/create-account-api-key", + "langsmith/set-up-organization", + "langsmith/set-up-workspace", + "langsmith/set-up-billing", + "langsmith/update-business-info", + "langsmith/set-up-access-control", + "langsmith/set-up-resource-tags", + "langsmith/manage-spend", + "langsmith/set-up-saml-sso", + "langsmith/set-up-scim" + ] + }, + { + "group": "Common data types", + "pages": [ + "langsmith/run-data-format", + "langsmith/feedback-data-format", + "langsmith/example-data-format", + "langsmith/trace-query-syntax", + "langsmith/dataset-json-types", + "langsmith/dataset-transformations" + ] + }, + { + "group": "Additional resources", + "pages": [ + "langsmith/faq", + "langsmith/troubleshooting", + "langsmith/cloud-architecture-and-scalability", + "langsmith/regions-faq", + "langsmith/authentication-methods" + ] + } + ] + } + ] + }, { "dropdown": "LangChain Labs", "icon": "/images/brand/langchain-labs-pill.svg", diff --git a/src/langsmith/dataset-subset.mdx b/src/langsmith/dataset-subset.mdx deleted file mode 100644 index b6c5c1762..000000000 --- a/src/langsmith/dataset-subset.mdx +++ /dev/null @@ -1,93 +0,0 @@ ---- -title: How to evaluate on a split / filtered view of a dataset -sidebarTitle: Evaluate on a split / filtered view of a dataset ---- - - - Before diving into this content, it might be helpful to read: - - * [guide on fetching examples](/langsmith/manage-datasets-programmatically#fetch-examples). - * [guide on creating/managing dataset splits](/langsmith/manage-datasets-in-application#create-and-manage-dataset-splits) - - -## Evaluate on a filtered view of a dataset - -You can use the `list_examples` / `listExamples` method to fetch a subset of examples from a dataset to evaluate on. You can refer to guide above to learn more about the different ways to fetch examples. - -One common workflow is to fetch examples that have a certain metadata key-value pair. - - - - -```python -from langsmith import evaluate - -results = evaluate( - lambda inputs: label_text(inputs["text"]), - data=client.list_examples(dataset_name=dataset_name, metadata={"desired_key": "desired_value"}), - evaluators=[correct_label], - experiment_prefix="Toxic Queries", -) -``` - - - - -```typescript -import { evaluate } from "langsmith/evaluation"; - -await evaluate((inputs) => labelText(inputs["input"]), { - data: langsmith.listExamples({ - datasetName: datasetName, - metadata: {"desired_key": "desired_value"}, - }), - evaluators: [correctLabel], - experimentPrefix: "Toxic Queries", -}); -``` - - - - -For more advanced filtering capabilities see this [how-to guide](/langsmith/manage-datasets-programmatically#list-examples-by-structured-filter). - -## Evaluate on a dataset split - -You can use the `list_examples` / `listExamples` method to evaluate on one or multiple splits of your dataset. The `splits` param takes a list of the splits you would like to evaluate. - - - - -```python -from langsmith import evaluate - -results = evaluate( - lambda inputs: label_text(inputs["text"]), - data=client.list_examples(dataset_name=dataset_name, splits=["test", "training"]), - evaluators=[correct_label], - experiment_prefix="Toxic Queries", -) -``` - - - - -```typescript -import { evaluate } from "langsmith/evaluation"; - -await evaluate((inputs) => labelText(inputs["input"]), { - data: langsmith.listExamples({ - datasetName: datasetName, - splits: ["test", "training"], - }), - evaluators: [correctLabel], - experimentPrefix: "Toxic Queries", -}); -``` - - - - -## Related - -* Learn more about how to fetch views of a dataset [here](/langsmith/manage-datasets-programmatically#fetch-datasets) diff --git a/src/langsmith/dataset-version.mdx b/src/langsmith/dataset-version.mdx deleted file mode 100644 index f34cf3e49..000000000 --- a/src/langsmith/dataset-version.mdx +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: How to evaluate on a specific dataset version -sidebarTitle: Evaluate on a specific dataset version ---- - - - Before diving into this content, it might be helpful to read the [guide on versioning datasets](/langsmith/version-datasets). Additionally, it might be helpful to read the [guide on fetching examples](/langsmith/manage-datasets-programmatically#fetch-examples). - - -## Using `list_examples` - -You can take advantage of the fact that `evaluate` / `aevaluate` allows passing in an iterable of examples to evaluate on a particular version of a dataset. Simply use `list_examples` / `listExamples` to fetch examples from a particular version tag using `as_of` / `asOf` and pass that in to the `data` argument. - - - - -```python -from langsmith import Client - -ls_client = Client() - -# Assumes actual outputs have a 'class' key. -# Assumes example outputs have a 'label' key. -def correct(outputs: dict, reference_outputs: dict) -> bool: - return outputs["class"] == reference_outputs["label"] - -results = ls_client.evaluate( - lambda inputs: {"class": "Not toxic"}, - # Pass in filtered data here: - data=ls_client.list_examples( - dataset_name="Toxic Queries", - as_of="latest", # specify version here - ), - evaluators=[correct], -) -``` - - - - -```typescript -import { evaluate } from "langsmith/evaluation"; - -await evaluate((inputs) => labelText(inputs["input"]), { - data: langsmith.listExamples({ - datasetName: datasetName, - asOf: "latest", - }), - evaluators: [correctLabel], -}); -``` - - - - -## Related - -* Learn more about how to fetch views of a dataset [here](/langsmith/manage-datasets-programmatically#fetch-datasets) diff --git a/src/langsmith/evaluation-overview.mdx b/src/langsmith/evaluation-overview.mdx index bbc2d0275..86df0f9b9 100644 --- a/src/langsmith/evaluation-overview.mdx +++ b/src/langsmith/evaluation-overview.mdx @@ -56,7 +56,7 @@ Learn how to [create and manage dataset splits](/langsmith/manage-datasets-in-ap ### Versions -Datasets are [versioned](/langsmith/version-datasets) such that every time you add, update, or delete examples in your dataset, a new version of the dataset is created. This makes it easy to inspect and revert changes to your dataset in case you make a mistake. You can also [tag versions](/langsmith/version-datasets#tag-a-version) of your dataset to give them a more human-readable name. This can be useful for marking important milestones in your dataset's history. +Datasets are [versioned](/langsmith/manage-datasets#version-a-dataset) such that every time you add, update, or delete examples in your dataset, a new version of the dataset is created. This makes it easy to inspect and revert changes to your dataset in case you make a mistake. You can also [tag versions](/langsmith/manage-datasets#tag-a-version) of your dataset to give them a more human-readable name. This can be useful for marking important milestones in your dataset's history. You can run evaluations on specific versions of a dataset. This can be useful when running evaluations in CI, to make sure that a dataset update doesn't accidentally break your CI pipelines. diff --git a/src/langsmith/export-filtered-traces-to-dataset.mdx b/src/langsmith/export-filtered-traces-to-dataset.mdx deleted file mode 100644 index 7b7141577..000000000 --- a/src/langsmith/export-filtered-traces-to-dataset.mdx +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: How to export filtered traces from experiment to dataset -sidebarTitle: Export filtered traces from experiment to dataset ---- - -After running an offline evaluation in LangSmith, you may want to export traces that met some evaluation criteria to a dataset. - -## View experiment traces - -![Export filtered traces](/langsmith/images/export-filtered-trace-to-dataset.png) - -To do so, first click on the arrow next to your experiment name. This will direct you to a project that contains the traces generated from your experiment. - -![Export filtered traces](/langsmith/images/experiment-tracing-project.png) - -From there, you can filter the traces based on your evaluation criteria. In this example, I want to filter for all traces that received an accuracy score greater than 0.5. - -![Export filtered traces](/langsmith/images/filtered-traces-from-experiment.png) - -Afte applying the filter on my project, I can multi-select runs I'd like to add to my dataset, and click the 'Add to Dataset' at the bottom of my screen. - -![Export filtered traces](/langsmith/images/add-filtered-traces-to-dataset.png) diff --git a/src/langsmith/manage-datasets-programmatically.mdx b/src/langsmith/manage-datasets-programmatically.mdx index 33abe0f31..be210071f 100644 --- a/src/langsmith/manage-datasets-programmatically.mdx +++ b/src/langsmith/manage-datasets-programmatically.mdx @@ -1,5 +1,5 @@ --- -title: How to manage datasets programmatically +title: How to create and manage datasets programmatically sidebarTitle: With the SDK --- diff --git a/src/langsmith/manage-datasets.mdx b/src/langsmith/manage-datasets.mdx new file mode 100644 index 000000000..d91965e69 --- /dev/null +++ b/src/langsmith/manage-datasets.mdx @@ -0,0 +1,258 @@ +--- +title: Manage datasets +sidebarTitle: Manage datasets +--- + +LangSmith provides tools for managing and working with your [_datasets_](/langsmith/evaluation-overview#datasets). This page describes dataset operations including: + +* [Filtering](#evaluate-on-a-filtered-view-of-a-dataset) and [splitting](#evaluate-on-a-dataset-split) datasets for evaluation. +* [Versioning datasets](#version-a-dataset) to track changes over time. +* [Sharing datasets](#share-a-dataset) publicly. +* [Exporting datasets](#export-a-dataset) in various formats. + +You'll also learn how to [export filtered traces](#export-filtered-traces-from-experiment-to-dataset) from [experiments](/langsmith/evaluation-overview#experiment) back to datasets for further analysis and iteration. + +## Evaluate on a split / filtered view of a dataset + + +You may find it helpful to refer to the following content before you read this section: + +- [Fetching examples](/langsmith/manage-datasets-programmatically#fetch-examples). +- [Creating and managing dataset splits](/langsmith/manage-datasets-in-application#create-and-manage-dataset-splits). + + +### Evaluate on a filtered view of a dataset + +You can use the `list_examples` / `listExamples` method to [fetch](/langsmith/manage-datasets-programmatically#fetch-examples) a subset of examples from a dataset to evaluate on. + +One common workflow is to fetch examples that have a certain metadata key-value pair. + + + + +```python +from langsmith import evaluate + +results = evaluate( + lambda inputs: label_text(inputs["text"]), + data=client.list_examples(dataset_name=dataset_name, metadata={"desired_key": "desired_value"}), + evaluators=[correct_label], + experiment_prefix="Toxic Queries", +) +``` + + + + +```typescript +import { evaluate } from "langsmith/evaluation"; + +await evaluate((inputs) => labelText(inputs["input"]), { + data: langsmith.listExamples({ + datasetName: datasetName, + metadata: {"desired_key": "desired_value"}, + }), + evaluators: [correctLabel], + experimentPrefix: "Toxic Queries", +}); +``` + + + + +For more filtering capabilities, refer to this [how-to guide](/langsmith/manage-datasets-programmatically#list-examples-by-structured-filter). + +### Evaluate on a dataset split + +You can use the `list_examples` / `listExamples` method to evaluate on one or multiple [splits](/langsmith/evaluation-overview#splits) of your dataset. The `splits` parameter takes a list of the splits you would like to evaluate. + + + + +```python +from langsmith import evaluate + +results = evaluate( + lambda inputs: label_text(inputs["text"]), + data=client.list_examples(dataset_name=dataset_name, splits=["test", "training"]), + evaluators=[correct_label], + experiment_prefix="Toxic Queries", +) +``` + + + + +```typescript +import { evaluate } from "langsmith/evaluation"; + +await evaluate((inputs) => labelText(inputs["input"]), { + data: langsmith.listExamples({ + datasetName: datasetName, + splits: ["test", "training"], + }), + evaluators: [correctLabel], + experimentPrefix: "Toxic Queries", +}); +``` + + + + +For more details on fetching views of a dataset, refer to the guide on [fetching datasets](/langsmith/manage-datasets-programmatically#fetch-datasets). + +## Version a dataset + +In LangSmith, datasets are versioned. This means that every time you add, update, or delete examples in your dataset, a new version of the dataset is created. + +### Create a new version of a dataset + +Any time you add, update, or delete examples in your dataset, a new [version](/langsmith/evaluation-overview#versions) of your dataset is created. This allows you to track changes to your dataset over time and understand how your dataset has evolved. + +By default, the version is defined by the timestamp of the change. When you click on a particular version of a dataset (by timestamp) in the **Examples** tab, you will find the state of the dataset at that point in time. + +![Version Datasets](/langsmith/images/version-dataset.png) + +Note that examples are read-only when viewing a past version of the dataset. You will also see the operations that were between this version of the dataset and the "latest" version of the dataset. + + +By default, the latest version of the dataset is shown in the **Examples** tab and experiments from all versions are shown in the **Tests** tab. + + +In the **Tests** tab, you will find the results of tests run on the dataset at different versions. + +![Version Datasets](/langsmith/images/version-dataset-tests.png) + +### Tag a version + +You can also tag versions of your dataset to give them a more human-readable name, which can be useful for marking important milestones in your dataset's history. + +For example, you might tag a version of your dataset as "prod" and use it to run tests against your LLM pipeline. + +You can tag a version of your dataset in the UI by clicking on **+ Tag this version** in the **Examples** tab. + +![Tagging Datasets](/langsmith/images/tag-this-version.png) + +You can also tag versions of your dataset using the SDK. Here's an example of how to tag a version of a dataset using the [Python SDK](https://docs.smith.langchain.com/reference/python/reference): + +```python +from langsmith import Client +from datetime import datetime + +client = Client() +initial_time = datetime(2024, 1, 1, 0, 0, 0) # The timestamp of the version you want to tag + +# You can tag a specific dataset version with a semantic name, like "prod" +client.update_dataset_tag( + dataset_name=toxic_dataset_name, as_of=initial_time, tag="prod" +) +``` + +To run an evaluation on a particular tagged version of a dataset, refer to the [Evaluate on a specific dataset version section](#evaluate-on-specific-dataset-version). + +## Evaluate on a specific dataset version + + +You may find it helpful to refer to the following content before you read this section: + +- [Version a dataset](#version-a-dataset). +- [Fetching examples](/langsmith/manage-datasets-programmatically#fetch-examples). + + +### Use `list_examples` + +You can use `evaluate` / `aevaluate` to pass in an iterable of examples to evaluate on a particular version of a dataset. Use `list_examples` / `listExamples` to fetch examples from a particular version tag using `as_of` / `asOf` and pass that into the `data` argument. + + + + +```python +from langsmith import Client + +ls_client = Client() + +# Assumes actual outputs have a 'class' key. +# Assumes example outputs have a 'label' key. +def correct(outputs: dict, reference_outputs: dict) -> bool: + return outputs["class"] == reference_outputs["label"] + +results = ls_client.evaluate( + lambda inputs: {"class": "Not toxic"}, + # Pass in filtered data here: + data=ls_client.list_examples( + dataset_name="Toxic Queries", + as_of="latest", # specify version here + ), + evaluators=[correct], +) +``` + + + + +```typescript +import { evaluate } from "langsmith/evaluation"; + +await evaluate((inputs) => labelText(inputs["input"]), { + data: langsmith.listExamples({ + datasetName: datasetName, + asOf: "latest", + }), + evaluators: [correctLabel], +}); +``` + + + + +Learn more about how to fetch views of a dataset on the [Create and manage datasets programmatically](/langsmith/manage-datasets-programmatically#fetch-datasets) page. + +## Share a dataset + +### Share a dataset publicly + + +Sharing a dataset publicly will make the **dataset examples, experiments and associated runs, and feedback on this dataset accessible to anyone with the link**, even if they don't have a LangSmith account. Make sure you're not sharing sensitive information. + +This feature is only available in the cloud-hosted version of LangSmith. + + +From the **Dataset & Experiments** tab, select a dataset, click **⋮** (top right of the page), click **Share Dataset**. This will open a dialog where you can copy the link to the dataset. + +![Share Dataset](/langsmith/images/share-dataset.gif) + +### Unshare a dataset + +1. Click on **Unshare** by click on **Public** in the upper right hand corner of any publicly shared dataset, then **Unshare** in the dialog. ![Unshare Dataset](/langsmith/images/unshare-dataset.png) + +2. Navigate to your organization's list of publicly shared datasets, by clicking on **Settings** -> **Shared URLs** or [this link](https://smith.langchain.com/settings/shared), then click on **Unshare** next to the dataset you want to unshare. ![Unshare Trace List](/langsmith/images/unshare-trace-list.png) + +## Export a dataset + +You can export your LangSmith dataset to a CSV, JSONL, or [OpenAI's fine tuning format](https://platform.openai.com/docs/guides/fine-tuning#example-format) from the LangSmith UI. + +From the **Dataset & Experiments** tab, select a dataset, click **⋮** (top right of the page), click **Download Dataset**. + +![Export Dataset Button](/langsmith/images/export-dataset-button.gif) + +## Export filtered traces from experiment to dataset + +After running an [offline evaluation](/langsmith/evaluation-overview#offline-evaluation) in LangSmith, you may want to export [traces](/langsmith/observability-overview#traces) that met some evaluation criteria to a dataset. + +### View experiment traces + +![Export filtered traces](/langsmith/images/export-filtered-trace-to-dataset.png) + +To do so, first click on the arrow next to your experiment name. This will direct you to a project that contains the traces generated from your experiment. + +![Export filtered traces](/langsmith/images/experiment-tracing-project.png) + +From there, you can filter the traces based on your evaluation criteria. In this example, we're filtering for all traces that received an accuracy score greater than 0.5. + +![Export filtered traces](/langsmith/images/filtered-traces-from-experiment.png) + +After applying the filter on the project, we can multi-select runs to add to the dataset, and click **Add to Dataset**. + +![Export filtered traces](/langsmith/images/add-filtered-traces-to-dataset.png) + + diff --git a/src/langsmith/manage-organization-by-api.mdx b/src/langsmith/manage-organization-by-api.mdx index 7e2b4fabd..e88ade251 100644 --- a/src/langsmith/manage-organization-by-api.mdx +++ b/src/langsmith/manage-organization-by-api.mdx @@ -70,7 +70,7 @@ Workspace level: ## Security Settings - "Shared resources" in this context refer to [public prompts](/langsmith/create-a-prompt#save-your-prompt), [shared runs](/langsmith/share-trace), and [shared datasets](/langsmith/share-dataset). + "Shared resources" in this context refer to [public prompts](/langsmith/create-a-prompt#save-your-prompt), [shared runs](/langsmith/share-trace), and [shared datasets](/langsmith/manage-datasets#share-a-dataset). * [Update organization sharing settings](https://api.smith.langchain.com/redoc#tag/orgs/operation/update_current_organization_info_api_v1_orgs_current_info_patch) diff --git a/src/langsmith/share-dataset.mdx b/src/langsmith/share-dataset.mdx deleted file mode 100644 index 5f97c5364..000000000 --- a/src/langsmith/share-dataset.mdx +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: How to share a dataset -sidebarTitle: Share a dataset ---- - -## Share a Dataset Publicly - - - Sharing a dataset publicly will make the **dataset examples, experiments and associated runs and feedback on this dataset accessible to anyone with the link**, even if they don't have a LangSmith account. Make sure you're not sharing sensitive information. - - This feature is only available in the cloud-hosted version of LangSmith. - - -From the **Dataset & Experiments** tab, select a dataset, click **⋮** (top right of the page), click **Share Dataset**. This will open a dialog where you can copy the link to the dataset. - -![Share Dataset](/langsmith/images/share-dataset.gif) - -**To "unshare" a dataset:** - -1. Click on **Unshare** by click on **Public** in the upper right hand corner of any publicly shared dataset, then **Unshare** in the dialog. ![Unshare Dataset](/langsmith/images/unshare-dataset.png) - -2. Navigate to your organization's list of publicly shared datasets, by clicking on **Settings** -> **Shared URLs** or [this link](https://smith.langchain.com/settings/shared), then click on **Unshare** next to the dataset you want to unshare. ![Unshare Trace List](/langsmith/images/unshare-trace-list.png) - -## Export a Dataset - -You can export your LangSmith dataset to a CSV, JSONL or [OpenAI's fine tuning format](https://platform.openai.com/docs/guides/fine-tuning#example-format) from the LangSmith UI. - -From the **Dataset & Experiments** tab, select a dataset, click **⋮** (top right of the page), click **Download Dataset**. - -![Export Dataset Button](/langsmith/images/export-dataset-button.gif) diff --git a/src/langsmith/version-datasets.mdx b/src/langsmith/version-datasets.mdx deleted file mode 100644 index 93b45e6a5..000000000 --- a/src/langsmith/version-datasets.mdx +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: How to version a dataset -sidebarTitle: Version a dataset ---- - -In LangSmith, datasets are versioned. This means that every time you add, update, or delete examples in your dataset, a new version of the dataset is created. - -## Create a new version of a dataset - -Any time you *add*, *update*, or *delete* examples in your dataset, a new version of your dataset is created. This allows you to track changes to your dataset over time and to understand how your dataset has evolved. - -By default, the version is defined by the timestamp of the change. When you click on a particular version of a dataset (by timestamp) in the "Examples" tab, you can see the state of the dataset at that point in time. - -![Version Datasets](/langsmith/images/version-dataset.png) - -Note that examples are read-only when viewing a past version of the dataset. You will also see the operations that were between this version of the dataset and the "latest" version of the dataset. Also, by default the **latest version of the dataset is shown in the "Examples" tab** and experiments from **all versions are shown in the "Tests" tab**. - -In the "Tests" tab, you can see the results of tests run on the dataset at different versions. - -![Version Datasets](/langsmith/images/version-dataset-tests.png) - -## Tag a version - -You can also tag versions of your dataset to give them a more human-readable name. This can be useful for marking important milestones in your dataset's history. - -For example, you might tag a version of your dataset as "prod" and use it to run tests against your LLM pipeline. - -Tagging can be done in the UI by clicking on "+ Tag this version" in the "Examples" tab. - -![Tagging Datasets](/langsmith/images/tag-this-version.png) - -You can also tag versions of your dataset using the SDK. Here's an example of how to tag a version of a dataset using the python SDK: - -```python -from langsmith import Client -from datetime import datetime - -client = Client() -initial_time = datetime(2024, 1, 1, 0, 0, 0) # The timestamp of the version you want to tag - -# You can tag a specific dataset version with a semantic name, like "prod" -client.update_dataset_tag( - dataset_name=toxic_dataset_name, as_of=initial_time, tag="prod" -) -``` - -To run an evaluation on a particular tagged version of a dataset, you can follow [this guide](/langsmith/dataset-version). From 0006f3048d57cdeafc8477a1dbc43442939a498a Mon Sep 17 00:00:00 2001 From: Kathryn May Date: Wed, 13 Aug 2025 17:17:38 -0400 Subject: [PATCH 2/4] docs: Consolidate the manage prompts dropdown pages --- src/docs.json | 9 +-- src/langsmith/langchain-hub.mdx | 18 ----- src/langsmith/manage-prompts.mdx | 114 ++++++++++++++++++++++++++++++ src/langsmith/prompt-tags.mdx | 51 ------------- src/langsmith/trigger-webhook.mdx | 47 ------------ 5 files changed, 115 insertions(+), 124 deletions(-) delete mode 100644 src/langsmith/langchain-hub.mdx create mode 100644 src/langsmith/manage-prompts.mdx delete mode 100644 src/langsmith/prompt-tags.mdx delete mode 100644 src/langsmith/trigger-webhook.mdx diff --git a/src/docs.json b/src/docs.json index 9076c90e6..d292a5a41 100644 --- a/src/docs.json +++ b/src/docs.json @@ -595,14 +595,7 @@ } ] }, - { - "group": "Manage prompts", - "pages": [ - "langsmith/prompt-tags", - "langsmith/trigger-webhook", - "langsmith/langchain-hub" - ] - }, + "langsmith/manage-prompts", { "group": "Tutorials", "pages": [ diff --git a/src/langsmith/langchain-hub.mdx b/src/langsmith/langchain-hub.mdx deleted file mode 100644 index e164e2fe8..000000000 --- a/src/langsmith/langchain-hub.mdx +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: Public prompt hub -sidebarTitle: Public prompt hub ---- - -LangSmith's public prompt hub is a collection of prompts that have been created by the LangChain community and can be used for reference. - - - Please note that prompts are user-generated and unverified. LangChain does not review or endorse public prompts, use these at your own risk. Use of Prompt Hub is subject to our [Terms of Service](https://www.langchain.com/terms-of-service). - - -Navigate to the **Prompts** section of the left-hand sidebar and click on **Browse all Public Prompts in the LangChain Hub**. - -Here you'll find all of the publicly listed prompts in the LangChain Hub. You can search for prompts by name, handle, use cases, descriptions, or models. You can fork prompts to your personal organization, view the prompt's details, and run the prompt in the playground. You can [pull any public prompt into your code](/langsmith/manage-prompts-programmatically) using the SDK. - -To view prompts tied to your workspace, visit the Prompts tab in the sidebar. - -![](/langsmith/images/prompts-tab.png) diff --git a/src/langsmith/manage-prompts.mdx b/src/langsmith/manage-prompts.mdx new file mode 100644 index 000000000..9f1d3862d --- /dev/null +++ b/src/langsmith/manage-prompts.mdx @@ -0,0 +1,114 @@ +--- +title: Manage prompts +sidebarTitle: Manage prompts +--- + +LangSmith provides several tools to help you manage your [_prompts_](/langsmith/prompt-engineering-overview) effectively. This page describes the following features: + +- [Commit tags](#commit-tags) for version control and environment management. +- [Webhook triggers](#trigger-a-webhook-on-prompt-commit) for automating workflows when prompts are updated. +- [Public prompt hub](#public-prompt-hub) for discovering and using community-created prompts. + +## Commit tags + +[_Commit tags_](/langsmith/prompt-engineering-overview#tags) are labels that reference a specific [_commit_](/langsmith/prompt-engineering-overview#commits) in your prompt's version history. They help you mark significant versions and control which versions run in different environments. By referencing tags rather than commit IDs in your code, you can update which version is being used without modifying the code itself. + +Each tag references exactly one commit, though you can reassign a tag to point to a different commit. + +Use cases for commit tags can include: + +- **Environment-specific tags**: Mark commits for `production` or `staging` environments, which allows you to switch between different versions without changing your code. +- **Version control**: Mark stable versions of your prompts, for example, `v1`, `v2`, which lets you reference specific versions in your code and track changes over time. +- **Collaboration**: Mark versions ready for review, which enables you to share specific versions with collaborators and get feedback. + +### Create a tag + +To create a tag, navigate to the **Commits** tab for a prompt. Click on the tag icon next to the commit you want to tag. Click **New Tag** and enter a name for the tag. + +![](/langsmith/images/commits-tab.png) ![](/langsmith/images/create-new-prompt-tag.png) + +### Move a tag + +To point a tag to a different commit, click on the tag icon next to the destination commit, and select the tag you want to move. This will automatically update the tag to point to the new commit. + +![](/langsmith/images/move-prompt-tag.png) + +### Delete a tag + +To delete a tag, click on the delete icon next to the tag you want to delete. This will delete the tag altogether and it will no longer be associated with any commit. + +### Use tags in code + +Tags provide a stable way to reference specific versions of your prompts in code. Instead of using commit hashes directly, you can reference tags that can be updated without changing your code. + +Here is an example of pulling a prompt by tag in Python: + +```python +prompt = client.pull_prompt("joke-generator:prod") +# If prod tag points to commit a1b2c3d4, this is equivalent to: +prompt = client.pull_prompt("joke-generator:a1b2c3d4") +``` + +For more information on how to use prompts in code, refer to [Managing prompts programmatically](/langsmith/manage-prompts-programmatically). + +## Trigger a webhook on prompt commit + +You can configure a webhook to be triggered whenever a commit is made to a prompt. + +Some common use cases of this include: + +* Triggering a CI/CD pipeline when prompts are updated. +* Synchronizing prompts with a GitHub repository. +* Notifying team members about prompt modifications. + +### Configure a webhook + +Navigate to the **Prompts** section in the left-hand sidebar or from the application homepage. In the top right corner, click on the `+ Webhook` button. + +Add a webhook URL and any required headers. + + +You can only configure one webhook per workspace. If you want to configure multiple per workspace or set up a different webhook for each prompt, let us know in the [LangChain Forum](https://forum.langchain.com/). + + +To test out your webhook, click the **Send test notification** button. This will send a test notification to the webhook URL you provided with a sample payload. + +The sample payload is a JSON object with the following fields: + +- `prompt_id`: The ID of the prompt that was committed. +- `prompt_name`: The name of the prompt that was committed. +- `commit_hash`: The commit hash of the prompt. +- `created_at`: The date of the commit. +- `created_by`: The author of the commit. +- `manifest`: The manifest of the prompt. + +### Trigger the webhook + +Commit to a prompt to trigger the webhook you've configured. + +#### Use the Playground + +If you do this in the Playground, you'll be prompted to deselect the webhooks you'd like to avoid triggering. + +![](/langsmith/images/commit-prompt-playground.png) + +#### Using the API + +If you commit via the API, you can specify to skip triggering the webhook by setting the `skip_webhooks` parameter to `true` or to an array of webhook ids to ignore. Refer to the [API docs](https://api.smith.langchain.com/redoc#tag/commits/operation/create_commit_api_v1_commits__owner___repo__post) for more information. + +## Public prompt hub + +LangSmith's public prompt hub is a collection of prompts that have been created by the LangChain community that you can use for reference. + + +Note that prompts are user-generated and unverified. LangChain does not review or endorse public prompts, use these at your own risk. Use of Prompt Hub is subject to our [Terms of Service](https://www.langchain.com/terms-of-service). + + +Navigate to the **Prompts** section of the left-hand sidebar and click on **Browse all Public Prompts in the LangChain Hub**. + +Here you'll find all of the publicly listed prompts in the LangChain Hub. You can search for prompts by name, handle, use cases, descriptions, or models. You can fork prompts to your personal organization, view the prompt's details, and run the prompt in the Playground. You can [pull any public prompt into your code](/langsmith/manage-prompts-programmatically) using the SDK. + +To view prompts tied to your workspace, visit the **Prompts** tab in the sidebar. + +![](/langsmith/images/prompts-tab.png) + diff --git a/src/langsmith/prompt-tags.mdx b/src/langsmith/prompt-tags.mdx deleted file mode 100644 index 441e83632..000000000 --- a/src/langsmith/prompt-tags.mdx +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Commit tags -sidebarTitle: Commit tags ---- - -Commit tags are labels that attached to specific commits in your prompt's version history. They help you mark significant versions and control which versions run in different environments. By referencing tags rather than commit IDs in your code, you can easily update which version is being used without modifying the code itself. - -## Overview - -* A tag is a named reference to a specific commit -* Each tag points to exactly one commit at a time -* Tags can be moved between commits -* Common uses include marking commits for different environments (e.g., "production", "staging") or marking stable versions - -## Managing Tags - -### Create a tag - -To create a tag, navigate to the commits tab of a prompt. Click on the tag icon next to the commit you want to tag. Click "New Tag" and enter the name of the tag. - -![](/langsmith/images/commits-tab.png) ![](/langsmith/images/create-new-prompt-tag.png) - -### Move a tag - -To point a tag to a different commit, click on the tag icon next to the destination commit, and select the tag you want to move. This will automatically update the tag to point to the new commit. - -![](/langsmith/images/move-prompt-tag.png) - -## Delete a tag - -To delete a tag, click on the delete icon next to the tag you want to delete. Note that this will delete the tag altogether and it will no longer be associated with any commit. - -## Using tags in code - -Tags provide a stable way to reference specific versions of your prompts in code. Instead of using commit hashes directly, you can reference tags which can be updated without changing your code. - -See [managing prompts programmatically](/langsmith/manage-prompts-programmatically) for more information on how to use prompts in code. - -Here is an example of pulling a prompt by tag in Python: - -```python -prompt = client.pull_prompt("joke-generator:prod") -# If prod tag points to commit a1b2c3d4, this is equivalent to: -prompt = client.pull_prompt("joke-generator:a1b2c3d4") -``` - -## Common use cases - -1. **Environment-specific tags**: Use tags like "prod" or "staging" to mark versions for different environments. This makes it easy to switch between different versions without changing your code. -2. **Version control**: Use tags to mark stable versions of your prompts (ex. v1, v2). This makes it easy to reference specific versions in your code and track changes over time. -3. **Collaboration**: Use tags to mark versions ready for review. This makes it easy to share specific versions with collaborators and get feedback. diff --git a/src/langsmith/trigger-webhook.mdx b/src/langsmith/trigger-webhook.mdx deleted file mode 100644 index 2c8980c41..000000000 --- a/src/langsmith/trigger-webhook.mdx +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: Trigger a webhook on prompt commit -sidebarTitle: Trigger a webhook on prompt commit ---- - -You can configure a webhook to be triggered whenever a commit is made to a prompt. - -Some common use cases of this include: - -* Triggering a CI/CD pipeline when prompts are updated -* Synchronizing prompts with a GitHub repository -* Notifying team members about prompt modifications - -## Configure a webhook - -Navigate to the **Prompts** section in the left-hand sidebar or from the application homepage. In the top right corner, click on the `+ Webhook` button. - -Add a webhook **url** and any required **headers**. - - - You can only configure one webhook per workspace. If you want to configure multiple per workspace or set up a different webhook for each prompt, let us know in the [LangChain Forum](https://forum.langchain.com/). - - -To test out your webhook, click the "Send test notification" button. This will send a test notification to the webhook url you provided with a sample payload. - -The sample payload is a JSON object with the following fields: - -* `prompt_id`: The id of the prompt that was committed. -* `prompt_name`: The name of the prompt that was committed. -* `commit_hash`: The commit hash of the prompt. -* `created_at`: The date of the commit. -* `created_by`: The author of the commit. -* `manifest`: The manifest of the prompt. - -## Trigger the webhook - -Commit to a prompt to trigger the webhook you've configured. - -### Using the Playground - -If you do this in the Playground, you'll be prompted to deselect the webhooks you'd like to avoid triggering. - -![](/langsmith/images/commit-prompt-playground.png) - -### Using the API - -If you commit via the API, you can specify to skip triggering the webhook by setting the `skip_webhooks` parameter to `true` or to an array of webhook ids to ignore. See API docs [here](https://api.smith.langchain.com/redoc#tag/commits/operation/create_commit_api_v1_commits__owner___repo__post) for more information. From 84b65b0e3c49c0cb7b40ceae8d24f4c5f015a376 Mon Sep 17 00:00:00 2001 From: Kathryn May Date: Fri, 15 Aug 2025 10:11:42 -0400 Subject: [PATCH 3/4] Update position of manage prompts page --- src/docs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs.json b/src/docs.json index d292a5a41..3c84585a1 100644 --- a/src/docs.json +++ b/src/docs.json @@ -581,6 +581,7 @@ "group": "Create and update prompts", "pages": [ "langsmith/create-a-prompt", + "langsmith/manage-prompts", "langsmith/manage-prompts-programmatically", "langsmith/managing-model-configurations", "langsmith/use-tools", @@ -595,7 +596,6 @@ } ] }, - "langsmith/manage-prompts", { "group": "Tutorials", "pages": [ From fa0354f3aac47205f5de8eb75c724ae47ab1d3a8 Mon Sep 17 00:00:00 2001 From: Kathryn May Date: Fri, 15 Aug 2025 10:16:09 -0400 Subject: [PATCH 4/4] Remove docs.json LS content for now --- src/docs.json | 401 -------------------------------------------------- 1 file changed, 401 deletions(-) diff --git a/src/docs.json b/src/docs.json index 3c84585a1..80169a115 100644 --- a/src/docs.json +++ b/src/docs.json @@ -316,407 +316,6 @@ } ] }, - { - "dropdown": "LangSmith", - "icon": "/images/brand/langsmith-pill.svg", - "description": "Platform for LLM observability and evaluation", - "tabs": [ - { - "tab": "Get started", - "pages": [ - "langsmith/home", - - { - "group": "Start tracing", - "pages": [ - "langsmith/observability-overview", - "langsmith/observability-quickstart" - ] - }, - { - "group": "Evaluate your application", - "pages": [ - "langsmith/evaluation-overview", - "langsmith/evaluation-quickstart" - ] - }, - { - "group": "Test your prompts", - "pages": [ - "langsmith/prompt-engineering-overview", - "langsmith/prompt-engineering-quickstart-ui", - "langsmith/prompt-engineering-quickstart-sdk" - ] - }, - { - "group": "API & SDKs", - "pages": [ - "langsmith/api-ref", - "langsmith/python-sdk", - "langsmith/js-ts-sdk" - ] - }, - { - "group": "Pricing", - "pages": [ - "langsmith/pricing-plans", - "langsmith/pricing-faq" - ] - } - ] - }, - { - "tab": "Observability", - "pages": [ - "langsmith/observability-tab", - { - "group": "Set up tracing", - "pages": [ - { - "group": "Manual", - "pages": [ - "langsmith/annotate-code", - "langsmith/trace-with-api" - ] - }, - { - "group": "Integrations", - "pages": [ - "langsmith/trace-with-langchain", - "langsmith/trace-with-langgraph", - "langsmith/trace-anthropic", - "langsmith/trace-openai", - "langsmith/trace-with-instructor", - "langsmith/trace-with-vercel-ai-sdk", - "langsmith/trace-with-opentelemetry", - "langsmith/trace-with-openai-agents-sdk", - "langsmith/trace-claude-code" - ] - }, - { - "group": "Configuration", - "pages": [ - "langsmith/log-traces-to-project", - "langsmith/trace-without-env-vars", - "langsmith/threads", - "langsmith/sample-traces", - "langsmith/add-metadata-tags", - "langsmith/distributed-tracing", - "langsmith/access-current-span", - "langsmith/log-llm-trace", - "langsmith/calculate-token-based-costs", - "langsmith/log-multimodal-traces", - "langsmith/log-retriever-trace", - "langsmith/mask-inputs-outputs", - "langsmith/trace-generator-functions", - "langsmith/serverless-environments", - "langsmith/nest-traces", - "langsmith/upload-files-with-traces", - "langsmith/output-detailed-logs", - "langsmith/troubleshooting-variable-caching", - "langsmith/collector-proxy" - ] - } - ] - }, - { - "group": "View traces", - "pages": [ - "langsmith/filter-traces-in-application", - "langsmith/export-traces", - "langsmith/data-export", - "langsmith/share-trace", - "langsmith/compare-traces", - "langsmith/langgraph-platform-logs" - ] - }, - { - "group": "Monitoring", - "pages": [ - "langsmith/dashboards", - "langsmith/alerts", - "langsmith/alerts-pagerduty", - "langsmith/alerts-webhook" - ] - }, - { - "group": "Automations", - "pages": [ - "langsmith/rules", - "langsmith/webhooks", - "langsmith/online-evaluations" - ] - }, - { - "group": "Human Feedback", - "pages": [ - "langsmith/attach-user-feedback" - ] - }, - { - "group": "Trace a RAG application", - "pages": [ - "langsmith/observability-llm-tutorial" - ] - } - ] - }, - { - "tab": "Evaluation", - "pages": [ - "langsmith/evaluation-tab", - { - "group": "Datasets", - "pages": [ - { - "group": "Create a dataset", - "pages": [ - "langsmith/manage-datasets-in-application", - "langsmith/manage-datasets-programmatically" - ] - }, - "langsmith/manage-datasets" - ] - }, - { - "group": "Evaluations", - "pages": [ - { - "group": "Run an evaluation", - "pages": [ - "langsmith/evaluate-llm-application", - "langsmith/run-evaluation-from-prompt-playground", - { - "group": "Prebuilt evaluators", - "pages": [ - "langsmith/prebuilt-evaluators" - ] - } - ] - }, - { - "group": "Evaluation methods", - "pages": [ - "langsmith/evaluation-async", - "langsmith/pytest", - "langsmith/vitest-jest", - "langsmith/run-evals-api-only" - ] - }, - { - "group": "Evaluator types", - "pages": [ - "langsmith/code-evaluator", - "langsmith/llm-as-judge", - "langsmith/summary", - "langsmith/evaluate-pairwise" - ] - }, - { - "group": "Evaluation techniques", - "pages": [ - "langsmith/define-target-function", - "langsmith/evaluate-on-intermediate-steps", - "langsmith/multiple-scores", - "langsmith/metric-type", - "langsmith/repetition", - "langsmith/rate-limiting", - "langsmith/local", - "langsmith/langchain-runnable", - "langsmith/evaluate-graph", - "langsmith/evaluate-existing-experiment", - "langsmith/evaluate-with-attachments", - "langsmith/multi-turn-simulation" - ] - }, - { - "group": "Improve evaluators", - "pages": [ - "langsmith/improve-judge-evaluator-feedback", - "langsmith/create-few-shot-evaluators", - "langsmith/index-datasets-for-dynamic-few-shot-example-selection" - ] - } - ] - }, - { - "group": "Analyze experiment results", - "pages": [ - "langsmith/analyze-single-experiment", - "langsmith/compare-experiment-results", - "langsmith/filter-experiments-ui", - "langsmith/fetch-perf-metrics-experiment", - "langsmith/upload-existing-experiments", - "langsmith/download-experiment-results-as-csv", - "langsmith/renaming-experiment", - "langsmith/bind-evaluator-to-dataset" - ] - }, - { - "group": "Annotation & human feedback", - "pages": [ - "langsmith/annotation-queues", - "langsmith/set-up-feedback-criteria", - "langsmith/annotate-traces-inline", - "langsmith/audit-evaluator-scores" - ] - }, - { - "group": "Tutorials", - "pages": [ - "langsmith/evaluate-chatbot-tutorial", - "langsmith/evaluate-rag-tutorial", - "langsmith/test-react-agent-pytest", - "langsmith/evaluate-complex-agent", - "langsmith/run-backtests-new-agent" - ] - } - ] - }, - { - "tab": "Prompt engineering", - "pages": [ - "langsmith/prompt-engineering-tab", - { - "group": "Create and update prompts", - "pages": [ - "langsmith/create-a-prompt", - "langsmith/manage-prompts", - "langsmith/manage-prompts-programmatically", - "langsmith/managing-model-configurations", - "langsmith/use-tools", - "langsmith/multimodal-content", - "langsmith/prompt-canvas", - { - "group": "Connect to models", - "pages": [ - "langsmith/custom-openai-compliant-model", - "langsmith/custom-endpoint" - ] - } - ] - }, - { - "group": "Tutorials", - "pages": [ - "langsmith/optimize-classifier", - "langsmith/prompt-commit", - "langsmith/multiple-messages" - ] - } - ] - }, - { - "tab": "Self-hosting", - "pages": [ - "langsmith/architectural-overview", - { - "group": "Setup", - "pages": [ - "langsmith/kubernetes", - "langsmith/docker", - "langsmith/self-host-usage", - "langsmith/self-host-upgrades", - "langsmith/self-host-egress", - "langsmith/self-host-organization-charts", - "langsmith/langsmith-managed-clickhouse" - ] - }, - { - "group": "Configuration", - "pages": [ - "langsmith/self-host-scale", - "langsmith/self-host-ttl", - "langsmith/self-host-ingress", - "langsmith/self-host-mirroring-images", - "langsmith/self-host-playground-environment-settings" - ] - }, - { - "group": "Authentication & access control", - "pages": [ - "langsmith/self-host-basic-auth", - "langsmith/self-host-sso", - "langsmith/self-host-user-management", - "langsmith/self-host-custom-tls-certificates", - "langsmith/self-host-using-an-existing-secret" - ] - }, - { - "group": "Connect external services", - "pages": [ - "langsmith/self-host-blob-storage", - "langsmith/self-host-external-clickhouse", - "langsmith/self-host-external-postgres", - "langsmith/self-host-external-redis" - ] - }, - { - "group": "Scripts", - "pages": [ - "langsmith/script-delete-a-workspace", - "langsmith/script-delete-an-organization", - "langsmith/script-delete-traces", - "langsmith/script-generate-clickhouse-stats", - "langsmith/script-generate-query-stats", - "langsmith/script-running-pg-support-queries", - "langsmith/script-running-ch-support-queries" - ] - }, - { - "group": "Observability", - "pages": [ - "langsmith/export-backend", - "langsmith/langsmith-collector", - "langsmith/observability-stack" - ] - } - ] - }, - { - "tab": "Administration", - "pages": [ - { - "group": "Setup", - "pages": [ - "langsmith/administration-overview", - "langsmith/create-account-api-key", - "langsmith/set-up-organization", - "langsmith/set-up-workspace", - "langsmith/set-up-billing", - "langsmith/update-business-info", - "langsmith/set-up-access-control", - "langsmith/set-up-resource-tags", - "langsmith/manage-spend", - "langsmith/set-up-saml-sso", - "langsmith/set-up-scim" - ] - }, - { - "group": "Common data types", - "pages": [ - "langsmith/run-data-format", - "langsmith/feedback-data-format", - "langsmith/example-data-format", - "langsmith/trace-query-syntax", - "langsmith/dataset-json-types", - "langsmith/dataset-transformations" - ] - }, - { - "group": "Additional resources", - "pages": [ - "langsmith/faq", - "langsmith/troubleshooting", - "langsmith/cloud-architecture-and-scalability", - "langsmith/regions-faq", - "langsmith/authentication-methods" - ] - } - ] - } - ] - }, { "dropdown": "LangChain Labs", "icon": "/images/brand/langchain-labs-pill.svg",