diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1e21f3fae..e853b86695 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,30 @@ jobs: - name: Run lints run: ./scripts/lint + upload: + if: github.repository == 'stainless-sdks/openai-python' + timeout-minutes: 10 + name: upload + permissions: + contents: read + id-token: write + runs-on: depot-ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Get GitHub OIDC Token + id: github-oidc + uses: actions/github-script@v6 + with: + script: core.setOutput('github_token', await core.getIDToken()); + + - name: Upload tarball + env: + URL: https://pkg.stainless.com/s + AUTH: ${{ steps.github-oidc.outputs.github_token }} + SHA: ${{ github.sha }} + run: ./scripts/utils/upload-artifact.sh + test: timeout-minutes: 10 name: test diff --git a/.release-please-manifest.json b/.release-please-manifest.json index f15af035f8..36925cfe97 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.78.1" + ".": "1.79.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 5f1bee851b..afa33d93bd 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 101 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-794a6ed3c3d3d77887564755168056af8a426b17cf1ec721e3a300503dc22a41.yml -openapi_spec_hash: 25a81c220713cd5b0bafc221d1dfa79a -config_hash: 0b768ed1b56c6d82816f0fa40dc4aaf5 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-262e171d0a8150ea1192474d16ba3afdf9a054b399f1a49a9c9b697a3073c136.yml +openapi_spec_hash: 33e00a48df8f94c94f46290c489f132b +config_hash: d8d5fda350f6db77c784f35429741a2e diff --git a/CHANGELOG.md b/CHANGELOG.md index b153f3ef05..9ec3e61533 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 1.79.0 (2025-05-16) + +Full Changelog: [v1.78.1...v1.79.0](https://github.com/openai/openai-python/compare/v1.78.1...v1.79.0) + +### Features + +* **api:** further updates for evals API ([32c99a6](https://github.com/openai/openai-python/commit/32c99a6f5885d4bf3511a7f06b70000edd274301)) +* **api:** manual updates ([25245e5](https://github.com/openai/openai-python/commit/25245e5e3d0713abfb65b760aee1f12bc61deb41)) +* **api:** responses x eval api ([fd586cb](https://github.com/openai/openai-python/commit/fd586cbdf889c9a5c6b9be177ff02fbfffa3eba5)) +* **api:** Updating Assistants and Evals API schemas ([98ba7d3](https://github.com/openai/openai-python/commit/98ba7d355551213a13803f68d5642eecbb4ffd39)) + + +### Bug Fixes + +* fix create audio transcription endpoint ([e9a89ab](https://github.com/openai/openai-python/commit/e9a89ab7b6387610e433550207a23973b7edda3a)) + + +### Chores + +* **ci:** fix installation instructions ([f26c5fc](https://github.com/openai/openai-python/commit/f26c5fc85d98d700b68cb55c8be5d15983a9aeaf)) +* **ci:** upload sdks to package manager ([861f105](https://github.com/openai/openai-python/commit/861f1055768168ab04987a42efcd32a07bc93542)) + ## 1.78.1 (2025-05-12) Full Changelog: [v1.78.0...v1.78.1](https://github.com/openai/openai-python/compare/v1.78.0...v1.78.1) diff --git a/pyproject.toml b/pyproject.toml index 71c86c38ea..5affe3c483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openai" -version = "1.78.1" +version = "1.79.0" description = "The official Python library for the openai API" dynamic = ["readme"] license = "Apache-2.0" diff --git a/scripts/utils/upload-artifact.sh b/scripts/utils/upload-artifact.sh new file mode 100755 index 0000000000..75198de98f --- /dev/null +++ b/scripts/utils/upload-artifact.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -exuo pipefail + +RESPONSE=$(curl -X POST "$URL" \ + -H "Authorization: Bearer $AUTH" \ + -H "Content-Type: application/json") + +SIGNED_URL=$(echo "$RESPONSE" | jq -r '.url') + +if [[ "$SIGNED_URL" == "null" ]]; then + echo -e "\033[31mFailed to get signed URL.\033[0m" + exit 1 +fi + +UPLOAD_RESPONSE=$(tar -cz . | curl -v -X PUT \ + -H "Content-Type: application/gzip" \ + --data-binary @- "$SIGNED_URL" 2>&1) + +if echo "$UPLOAD_RESPONSE" | grep -q "HTTP/[0-9.]* 200"; then + echo -e "\033[32mUploaded build to Stainless storage.\033[0m" + echo -e "\033[32mInstallation: pip install 'https://pkg.stainless.com/s/openai-python/$SHA'\033[0m" +else + echo -e "\033[31mFailed to upload artifact.\033[0m" + exit 1 +fi diff --git a/src/openai/_version.py b/src/openai/_version.py index 9b430dfa8b..77c73cdfd9 100644 --- a/src/openai/_version.py +++ b/src/openai/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "openai" -__version__ = "1.78.1" # x-release-please-version +__version__ = "1.79.0" # x-release-please-version diff --git a/src/openai/resources/audio/transcriptions.py b/src/openai/resources/audio/transcriptions.py index 0c7ebca7a6..bca8210a83 100644 --- a/src/openai/resources/audio/transcriptions.py +++ b/src/openai/resources/audio/transcriptions.py @@ -57,6 +57,7 @@ def create( *, file: FileTypes, model: Union[str, AudioModel], + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, response_format: Union[Literal["json"], NotGiven] = NOT_GIVEN, language: str | NotGiven = NOT_GIVEN, @@ -77,6 +78,7 @@ def create( *, file: FileTypes, model: Union[str, AudioModel], + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, response_format: Literal["verbose_json"], language: str | NotGiven = NOT_GIVEN, @@ -97,6 +99,7 @@ def create( *, file: FileTypes, model: Union[str, AudioModel], + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, response_format: Literal["text", "srt", "vtt"], include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, language: str | NotGiven = NOT_GIVEN, @@ -118,6 +121,7 @@ def create( file: FileTypes, model: Union[str, AudioModel], stream: Literal[True], + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, language: str | NotGiven = NOT_GIVEN, prompt: str | NotGiven = NOT_GIVEN, @@ -152,6 +156,11 @@ def create( Note: Streaming is not supported for the `whisper-1` model and will be ignored. + chunking_strategy: Controls how the audio is cut into chunks. When set to `"auto"`, the server + first normalizes loudness and then uses voice activity detection (VAD) to choose + boundaries. `server_vad` object can be provided to tweak VAD detection + parameters manually. If unset, the audio is transcribed as a single block. + include: Additional information to include in the transcription response. `logprobs` will return the log probabilities of the tokens in the response to understand the model's confidence in the transcription. `logprobs` only works with @@ -200,6 +209,7 @@ def create( file: FileTypes, model: Union[str, AudioModel], stream: bool, + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, language: str | NotGiven = NOT_GIVEN, prompt: str | NotGiven = NOT_GIVEN, @@ -234,6 +244,11 @@ def create( Note: Streaming is not supported for the `whisper-1` model and will be ignored. + chunking_strategy: Controls how the audio is cut into chunks. When set to `"auto"`, the server + first normalizes loudness and then uses voice activity detection (VAD) to choose + boundaries. `server_vad` object can be provided to tweak VAD detection + parameters manually. If unset, the audio is transcribed as a single block. + include: Additional information to include in the transcription response. `logprobs` will return the log probabilities of the tokens in the response to understand the model's confidence in the transcription. `logprobs` only works with @@ -281,6 +296,7 @@ def create( *, file: FileTypes, model: Union[str, AudioModel], + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, language: str | NotGiven = NOT_GIVEN, prompt: str | NotGiven = NOT_GIVEN, @@ -299,6 +315,7 @@ def create( { "file": file, "model": model, + "chunking_strategy": chunking_strategy, "include": include, "language": language, "prompt": prompt, @@ -357,11 +374,13 @@ async def create( *, file: FileTypes, model: Union[str, AudioModel], - response_format: Union[Literal["json"], NotGiven] = NOT_GIVEN, + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, + include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, language: str | NotGiven = NOT_GIVEN, prompt: str | NotGiven = NOT_GIVEN, + response_format: Union[Literal["json"], NotGiven] = NOT_GIVEN, + stream: Optional[Literal[False]] | NotGiven = NOT_GIVEN, temperature: float | NotGiven = NOT_GIVEN, - include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, timestamp_granularities: List[Literal["word", "segment"]] | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -369,7 +388,68 @@ async def create( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> Transcription: ... + ) -> TranscriptionCreateResponse: + """ + Transcribes audio into the input language. + + Args: + file: + The audio file object (not file name) to transcribe, in one of these formats: + flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm. + + model: ID of the model to use. The options are `gpt-4o-transcribe`, + `gpt-4o-mini-transcribe`, and `whisper-1` (which is powered by our open source + Whisper V2 model). + + chunking_strategy: Controls how the audio is cut into chunks. When set to `"auto"`, the server + first normalizes loudness and then uses voice activity detection (VAD) to choose + boundaries. `server_vad` object can be provided to tweak VAD detection + parameters manually. If unset, the audio is transcribed as a single block. + + include: Additional information to include in the transcription response. `logprobs` will + return the log probabilities of the tokens in the response to understand the + model's confidence in the transcription. `logprobs` only works with + response_format set to `json` and only with the models `gpt-4o-transcribe` and + `gpt-4o-mini-transcribe`. + + language: The language of the input audio. Supplying the input language in + [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (e.g. `en`) + format will improve accuracy and latency. + + prompt: An optional text to guide the model's style or continue a previous audio + segment. The + [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting) + should match the audio language. + + response_format: The format of the output, in one of these options: `json`, `text`, `srt`, + `verbose_json`, or `vtt`. For `gpt-4o-transcribe` and `gpt-4o-mini-transcribe`, + the only supported format is `json`. + + stream: If set to true, the model response data will be streamed to the client as it is + generated using + [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + See the + [Streaming section of the Speech-to-Text guide](https://platform.openai.com/docs/guides/speech-to-text?lang=curl#streaming-transcriptions) + for more information. + + Note: Streaming is not supported for the `whisper-1` model and will be ignored. + + temperature: The sampling temperature, between 0 and 1. Higher values like 0.8 will make the + output more random, while lower values like 0.2 will make it more focused and + deterministic. If set to 0, the model will use + [log probability](https://en.wikipedia.org/wiki/Log_probability) to + automatically increase the temperature until certain thresholds are hit. + + timestamp_granularities: The timestamp granularities to populate for this transcription. + `response_format` must be set `verbose_json` to use timestamp granularities. + Either or both of these options are supported: `word`, or `segment`. Note: There + is no additional latency for segment timestamps, but generating word timestamps + incurs additional latency. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + """ @overload async def create( @@ -377,6 +457,7 @@ async def create( *, file: FileTypes, model: Union[str, AudioModel], + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, response_format: Literal["verbose_json"], language: str | NotGiven = NOT_GIVEN, @@ -397,6 +478,7 @@ async def create( *, file: FileTypes, model: Union[str, AudioModel], + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, response_format: Literal["text", "srt", "vtt"], language: str | NotGiven = NOT_GIVEN, @@ -418,6 +500,7 @@ async def create( file: FileTypes, model: Union[str, AudioModel], stream: Literal[True], + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, language: str | NotGiven = NOT_GIVEN, prompt: str | NotGiven = NOT_GIVEN, @@ -452,6 +535,11 @@ async def create( Note: Streaming is not supported for the `whisper-1` model and will be ignored. + chunking_strategy: Controls how the audio is cut into chunks. When set to `"auto"`, the server + first normalizes loudness and then uses voice activity detection (VAD) to choose + boundaries. `server_vad` object can be provided to tweak VAD detection + parameters manually. If unset, the audio is transcribed as a single block. + include: Additional information to include in the transcription response. `logprobs` will return the log probabilities of the tokens in the response to understand the model's confidence in the transcription. `logprobs` only works with @@ -500,6 +588,7 @@ async def create( file: FileTypes, model: Union[str, AudioModel], stream: bool, + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, language: str | NotGiven = NOT_GIVEN, prompt: str | NotGiven = NOT_GIVEN, @@ -534,6 +623,11 @@ async def create( Note: Streaming is not supported for the `whisper-1` model and will be ignored. + chunking_strategy: Controls how the audio is cut into chunks. When set to `"auto"`, the server + first normalizes loudness and then uses voice activity detection (VAD) to choose + boundaries. `server_vad` object can be provided to tweak VAD detection + parameters manually. If unset, the audio is transcribed as a single block. + include: Additional information to include in the transcription response. `logprobs` will return the log probabilities of the tokens in the response to understand the model's confidence in the transcription. `logprobs` only works with @@ -581,6 +675,7 @@ async def create( *, file: FileTypes, model: Union[str, AudioModel], + chunking_strategy: Optional[transcription_create_params.ChunkingStrategy] | NotGiven = NOT_GIVEN, include: List[TranscriptionInclude] | NotGiven = NOT_GIVEN, language: str | NotGiven = NOT_GIVEN, prompt: str | NotGiven = NOT_GIVEN, @@ -599,6 +694,7 @@ async def create( { "file": file, "model": model, + "chunking_strategy": chunking_strategy, "include": include, "language": language, "prompt": prompt, diff --git a/src/openai/resources/embeddings.py b/src/openai/resources/embeddings.py index a392d5eb17..553dacc284 100644 --- a/src/openai/resources/embeddings.py +++ b/src/openai/resources/embeddings.py @@ -66,11 +66,12 @@ def create( input: Input text to embed, encoded as a string or array of tokens. To embed multiple inputs in a single request, pass an array of strings or array of token arrays. The input must not exceed the max input tokens for the model (8192 tokens for - `text-embedding-ada-002`), cannot be an empty string, and any array must be 2048 + all embedding models), cannot be an empty string, and any array must be 2048 dimensions or less. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) - for counting tokens. Some models may also impose a limit on total number of - tokens summed across inputs. + for counting tokens. In addition to the per-input token limit, all embedding + models enforce a maximum of 300,000 tokens summed across all inputs in a single + request. model: ID of the model to use. You can use the [List models](https://platform.openai.com/docs/api-reference/models/list) API to @@ -181,11 +182,12 @@ async def create( input: Input text to embed, encoded as a string or array of tokens. To embed multiple inputs in a single request, pass an array of strings or array of token arrays. The input must not exceed the max input tokens for the model (8192 tokens for - `text-embedding-ada-002`), cannot be an empty string, and any array must be 2048 + all embedding models), cannot be an empty string, and any array must be 2048 dimensions or less. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) - for counting tokens. Some models may also impose a limit on total number of - tokens summed across inputs. + for counting tokens. In addition to the per-input token limit, all embedding + models enforce a maximum of 300,000 tokens summed across all inputs in a single + request. model: ID of the model to use. You can use the [List models](https://platform.openai.com/docs/api-reference/models/list) API to diff --git a/src/openai/resources/evals/evals.py b/src/openai/resources/evals/evals.py index c12562a86d..7aba192c51 100644 --- a/src/openai/resources/evals/evals.py +++ b/src/openai/resources/evals/evals.py @@ -74,15 +74,20 @@ def create( ) -> EvalCreateResponse: """ Create the structure of an evaluation that can be used to test a model's - performance. An evaluation is a set of testing criteria and a datasource. After + performance. An evaluation is a set of testing criteria and the config for a + data source, which dictates the schema of the data used in the evaluation. After creating an evaluation, you can run it on different models and model parameters. We support several types of graders and datasources. For more information, see the [Evals guide](https://platform.openai.com/docs/guides/evals). Args: - data_source_config: The configuration for the data source used for the evaluation runs. + data_source_config: The configuration for the data source used for the evaluation runs. Dictates the + schema of the data used in the evaluation. - testing_criteria: A list of graders for all eval runs in this group. + testing_criteria: A list of graders for all eval runs in this group. Graders can reference + variables in the data source using double curly braces notation, like + `{{item.variable_name}}`. To reference the model's output, use the `sample` + namespace (ie, `{{sample.output_text}}`). metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful for storing additional information about the object in a structured format, and @@ -333,15 +338,20 @@ async def create( ) -> EvalCreateResponse: """ Create the structure of an evaluation that can be used to test a model's - performance. An evaluation is a set of testing criteria and a datasource. After + performance. An evaluation is a set of testing criteria and the config for a + data source, which dictates the schema of the data used in the evaluation. After creating an evaluation, you can run it on different models and model parameters. We support several types of graders and datasources. For more information, see the [Evals guide](https://platform.openai.com/docs/guides/evals). Args: - data_source_config: The configuration for the data source used for the evaluation runs. + data_source_config: The configuration for the data source used for the evaluation runs. Dictates the + schema of the data used in the evaluation. - testing_criteria: A list of graders for all eval runs in this group. + testing_criteria: A list of graders for all eval runs in this group. Graders can reference + variables in the data source using double curly braces notation, like + `{{item.variable_name}}`. To reference the model's output, use the `sample` + namespace (ie, `{{sample.output_text}}`). metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful for storing additional information about the object in a structured format, and diff --git a/src/openai/resources/evals/runs/runs.py b/src/openai/resources/evals/runs/runs.py index d74c91e3c4..7efc61292c 100644 --- a/src/openai/resources/evals/runs/runs.py +++ b/src/openai/resources/evals/runs/runs.py @@ -72,9 +72,10 @@ def create( extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, ) -> RunCreateResponse: - """Create a new evaluation run. - - This is the endpoint that will kick off grading. + """ + Kicks off a new run for a given evaluation, specifying the data source, and what + model configuration to use to test. The datasource will be validated against the + schema specified in the config of the evaluation. Args: data_source: Details about the run's data source. @@ -321,9 +322,10 @@ async def create( extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, ) -> RunCreateResponse: - """Create a new evaluation run. - - This is the endpoint that will kick off grading. + """ + Kicks off a new run for a given evaluation, specifying the data source, and what + model configuration to use to test. The datasource will be validated against the + schema specified in the config of the evaluation. Args: data_source: Details about the run's data source. diff --git a/src/openai/types/audio/transcription_create_params.py b/src/openai/types/audio/transcription_create_params.py index 0cda4c7907..8271b054ab 100644 --- a/src/openai/types/audio/transcription_create_params.py +++ b/src/openai/types/audio/transcription_create_params.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import List, Union, Optional -from typing_extensions import Literal, Required, TypedDict +from typing_extensions import Literal, Required, TypeAlias, TypedDict from ..._types import FileTypes from ..audio_model import AudioModel @@ -12,6 +12,8 @@ __all__ = [ "TranscriptionCreateParamsBase", + "ChunkingStrategy", + "ChunkingStrategyVadConfig", "TranscriptionCreateParamsNonStreaming", "TranscriptionCreateParamsStreaming", ] @@ -31,6 +33,15 @@ class TranscriptionCreateParamsBase(TypedDict, total=False): (which is powered by our open source Whisper V2 model). """ + chunking_strategy: Optional[ChunkingStrategy] + """Controls how the audio is cut into chunks. + + When set to `"auto"`, the server first normalizes loudness and then uses voice + activity detection (VAD) to choose boundaries. `server_vad` object can be + provided to tweak VAD detection parameters manually. If unset, the audio is + transcribed as a single block. + """ + include: List[TranscriptionInclude] """Additional information to include in the transcription response. @@ -82,6 +93,31 @@ class TranscriptionCreateParamsBase(TypedDict, total=False): """ +class ChunkingStrategyVadConfig(TypedDict, total=False): + type: Required[Literal["server_vad"]] + """Must be set to `server_vad` to enable manual chunking using server side VAD.""" + + prefix_padding_ms: int + """Amount of audio to include before the VAD detected speech (in milliseconds).""" + + silence_duration_ms: int + """ + Duration of silence to detect speech stop (in milliseconds). With shorter values + the model will respond more quickly, but may jump in on short pauses from the + user. + """ + + threshold: float + """Sensitivity threshold (0.0 to 1.0) for voice activity detection. + + A higher threshold will require louder audio to activate the model, and thus + might perform better in noisy environments. + """ + + +ChunkingStrategy: TypeAlias = Union[Literal["auto"], ChunkingStrategyVadConfig] + + class TranscriptionCreateParamsNonStreaming(TranscriptionCreateParamsBase, total=False): stream: Optional[Literal[False]] """ diff --git a/src/openai/types/beta/realtime/transcription_session_updated_event.py b/src/openai/types/beta/realtime/transcription_session_updated_event.py index ffc100bcc2..1f1fbdae14 100644 --- a/src/openai/types/beta/realtime/transcription_session_updated_event.py +++ b/src/openai/types/beta/realtime/transcription_session_updated_event.py @@ -16,7 +16,7 @@ class TranscriptionSessionUpdatedEvent(BaseModel): """A new Realtime transcription session configuration. When a session is created on the server via REST API, the session object also - contains an ephemeral key. Default TTL for keys is one minute. This property is + contains an ephemeral key. Default TTL for keys is 10 minutes. This property is not present when a session is updated via the WebSocket API. """ diff --git a/src/openai/types/embedding_create_params.py b/src/openai/types/embedding_create_params.py index a90566449b..94edce10a4 100644 --- a/src/openai/types/embedding_create_params.py +++ b/src/openai/types/embedding_create_params.py @@ -16,11 +16,12 @@ class EmbeddingCreateParams(TypedDict, total=False): To embed multiple inputs in a single request, pass an array of strings or array of token arrays. The input must not exceed the max input tokens for the model - (8192 tokens for `text-embedding-ada-002`), cannot be an empty string, and any - array must be 2048 dimensions or less. + (8192 tokens for all embedding models), cannot be an empty string, and any array + must be 2048 dimensions or less. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) - for counting tokens. Some models may also impose a limit on total number of - tokens summed across inputs. + for counting tokens. In addition to the per-input token limit, all embedding + models enforce a maximum of 300,000 tokens summed across all inputs in a single + request. """ model: Required[Union[str, EmbeddingModel]] diff --git a/src/openai/types/eval_create_params.py b/src/openai/types/eval_create_params.py index 3b712580a0..20a3765481 100644 --- a/src/openai/types/eval_create_params.py +++ b/src/openai/types/eval_create_params.py @@ -16,6 +16,7 @@ "EvalCreateParams", "DataSourceConfig", "DataSourceConfigCustom", + "DataSourceConfigLogs", "DataSourceConfigStoredCompletions", "TestingCriterion", "TestingCriterionLabelModel", @@ -32,10 +33,18 @@ class EvalCreateParams(TypedDict, total=False): data_source_config: Required[DataSourceConfig] - """The configuration for the data source used for the evaluation runs.""" + """The configuration for the data source used for the evaluation runs. + + Dictates the schema of the data used in the evaluation. + """ testing_criteria: Required[Iterable[TestingCriterion]] - """A list of graders for all eval runs in this group.""" + """A list of graders for all eval runs in this group. + + Graders can reference variables in the data source using double curly braces + notation, like `{{item.variable_name}}`. To reference the model's output, use + the `sample` namespace (ie, `{{sample.output_text}}`). + """ metadata: Optional[Metadata] """Set of 16 key-value pairs that can be attached to an object. @@ -65,6 +74,14 @@ class DataSourceConfigCustom(TypedDict, total=False): """ +class DataSourceConfigLogs(TypedDict, total=False): + type: Required[Literal["logs"]] + """The type of data source. Always `logs`.""" + + metadata: Dict[str, object] + """Metadata filters for the logs data source.""" + + class DataSourceConfigStoredCompletions(TypedDict, total=False): type: Required[Literal["stored_completions"]] """The type of data source. Always `stored_completions`.""" @@ -73,7 +90,7 @@ class DataSourceConfigStoredCompletions(TypedDict, total=False): """Metadata filters for the stored completions data source.""" -DataSourceConfig: TypeAlias = Union[DataSourceConfigCustom, DataSourceConfigStoredCompletions] +DataSourceConfig: TypeAlias = Union[DataSourceConfigCustom, DataSourceConfigLogs, DataSourceConfigStoredCompletions] class TestingCriterionLabelModelInputSimpleInputMessage(TypedDict, total=False): @@ -120,7 +137,7 @@ class TestingCriterionLabelModel(TypedDict, total=False): input: Required[Iterable[TestingCriterionLabelModelInput]] """A list of chat messages forming the prompt or context. - May include variable references to the "item" namespace, ie {{item.name}}. + May include variable references to the `item` namespace, ie {{item.name}}. """ labels: Required[List[str]] diff --git a/src/openai/types/eval_create_response.py b/src/openai/types/eval_create_response.py index d5f158ad29..20b0e3127f 100644 --- a/src/openai/types/eval_create_response.py +++ b/src/openai/types/eval_create_response.py @@ -1,8 +1,10 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import List, Union, Optional +from typing import Dict, List, Union, Optional from typing_extensions import Literal, Annotated, TypeAlias +from pydantic import Field as FieldInfo + from .._utils import PropertyInfo from .._models import BaseModel from .shared.metadata import Metadata @@ -17,14 +19,38 @@ __all__ = [ "EvalCreateResponse", "DataSourceConfig", + "DataSourceConfigLogs", "TestingCriterion", "TestingCriterionEvalGraderTextSimilarity", "TestingCriterionEvalGraderPython", "TestingCriterionEvalGraderScoreModel", ] + +class DataSourceConfigLogs(BaseModel): + schema_: Dict[str, object] = FieldInfo(alias="schema") + """ + The json schema for the run data source items. Learn how to build JSON schemas + [here](https://json-schema.org/). + """ + + type: Literal["logs"] + """The type of data source. Always `logs`.""" + + metadata: Optional[Metadata] = None + """Set of 16 key-value pairs that can be attached to an object. + + This can be useful for storing additional information about the object in a + structured format, and querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + """ + + DataSourceConfig: TypeAlias = Annotated[ - Union[EvalCustomDataSourceConfig, EvalStoredCompletionsDataSourceConfig], PropertyInfo(discriminator="type") + Union[EvalCustomDataSourceConfig, DataSourceConfigLogs, EvalStoredCompletionsDataSourceConfig], + PropertyInfo(discriminator="type"), ] diff --git a/src/openai/types/eval_list_response.py b/src/openai/types/eval_list_response.py index b743f57f6a..5ac4997cf6 100644 --- a/src/openai/types/eval_list_response.py +++ b/src/openai/types/eval_list_response.py @@ -1,8 +1,10 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import List, Union, Optional +from typing import Dict, List, Union, Optional from typing_extensions import Literal, Annotated, TypeAlias +from pydantic import Field as FieldInfo + from .._utils import PropertyInfo from .._models import BaseModel from .shared.metadata import Metadata @@ -17,14 +19,38 @@ __all__ = [ "EvalListResponse", "DataSourceConfig", + "DataSourceConfigLogs", "TestingCriterion", "TestingCriterionEvalGraderTextSimilarity", "TestingCriterionEvalGraderPython", "TestingCriterionEvalGraderScoreModel", ] + +class DataSourceConfigLogs(BaseModel): + schema_: Dict[str, object] = FieldInfo(alias="schema") + """ + The json schema for the run data source items. Learn how to build JSON schemas + [here](https://json-schema.org/). + """ + + type: Literal["logs"] + """The type of data source. Always `logs`.""" + + metadata: Optional[Metadata] = None + """Set of 16 key-value pairs that can be attached to an object. + + This can be useful for storing additional information about the object in a + structured format, and querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + """ + + DataSourceConfig: TypeAlias = Annotated[ - Union[EvalCustomDataSourceConfig, EvalStoredCompletionsDataSourceConfig], PropertyInfo(discriminator="type") + Union[EvalCustomDataSourceConfig, DataSourceConfigLogs, EvalStoredCompletionsDataSourceConfig], + PropertyInfo(discriminator="type"), ] diff --git a/src/openai/types/eval_retrieve_response.py b/src/openai/types/eval_retrieve_response.py index dabb20674e..758f9cc040 100644 --- a/src/openai/types/eval_retrieve_response.py +++ b/src/openai/types/eval_retrieve_response.py @@ -1,8 +1,10 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import List, Union, Optional +from typing import Dict, List, Union, Optional from typing_extensions import Literal, Annotated, TypeAlias +from pydantic import Field as FieldInfo + from .._utils import PropertyInfo from .._models import BaseModel from .shared.metadata import Metadata @@ -17,14 +19,38 @@ __all__ = [ "EvalRetrieveResponse", "DataSourceConfig", + "DataSourceConfigLogs", "TestingCriterion", "TestingCriterionEvalGraderTextSimilarity", "TestingCriterionEvalGraderPython", "TestingCriterionEvalGraderScoreModel", ] + +class DataSourceConfigLogs(BaseModel): + schema_: Dict[str, object] = FieldInfo(alias="schema") + """ + The json schema for the run data source items. Learn how to build JSON schemas + [here](https://json-schema.org/). + """ + + type: Literal["logs"] + """The type of data source. Always `logs`.""" + + metadata: Optional[Metadata] = None + """Set of 16 key-value pairs that can be attached to an object. + + This can be useful for storing additional information about the object in a + structured format, and querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + """ + + DataSourceConfig: TypeAlias = Annotated[ - Union[EvalCustomDataSourceConfig, EvalStoredCompletionsDataSourceConfig], PropertyInfo(discriminator="type") + Union[EvalCustomDataSourceConfig, DataSourceConfigLogs, EvalStoredCompletionsDataSourceConfig], + PropertyInfo(discriminator="type"), ] diff --git a/src/openai/types/eval_update_response.py b/src/openai/types/eval_update_response.py index c5cb2622ea..3f0b90ae03 100644 --- a/src/openai/types/eval_update_response.py +++ b/src/openai/types/eval_update_response.py @@ -1,8 +1,10 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import List, Union, Optional +from typing import Dict, List, Union, Optional from typing_extensions import Literal, Annotated, TypeAlias +from pydantic import Field as FieldInfo + from .._utils import PropertyInfo from .._models import BaseModel from .shared.metadata import Metadata @@ -17,14 +19,38 @@ __all__ = [ "EvalUpdateResponse", "DataSourceConfig", + "DataSourceConfigLogs", "TestingCriterion", "TestingCriterionEvalGraderTextSimilarity", "TestingCriterionEvalGraderPython", "TestingCriterionEvalGraderScoreModel", ] + +class DataSourceConfigLogs(BaseModel): + schema_: Dict[str, object] = FieldInfo(alias="schema") + """ + The json schema for the run data source items. Learn how to build JSON schemas + [here](https://json-schema.org/). + """ + + type: Literal["logs"] + """The type of data source. Always `logs`.""" + + metadata: Optional[Metadata] = None + """Set of 16 key-value pairs that can be attached to an object. + + This can be useful for storing additional information about the object in a + structured format, and querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + """ + + DataSourceConfig: TypeAlias = Annotated[ - Union[EvalCustomDataSourceConfig, EvalStoredCompletionsDataSourceConfig], PropertyInfo(discriminator="type") + Union[EvalCustomDataSourceConfig, DataSourceConfigLogs, EvalStoredCompletionsDataSourceConfig], + PropertyInfo(discriminator="type"), ] diff --git a/src/openai/types/evals/create_eval_completions_run_data_source.py b/src/openai/types/evals/create_eval_completions_run_data_source.py index 29c687b542..064ef3a310 100644 --- a/src/openai/types/evals/create_eval_completions_run_data_source.py +++ b/src/openai/types/evals/create_eval_completions_run_data_source.py @@ -117,7 +117,7 @@ class InputMessagesTemplate(BaseModel): template: List[InputMessagesTemplateTemplate] """A list of chat messages forming the prompt or context. - May include variable references to the "item" namespace, ie {{item.name}}. + May include variable references to the `item` namespace, ie {{item.name}}. """ type: Literal["template"] @@ -126,7 +126,7 @@ class InputMessagesTemplate(BaseModel): class InputMessagesItemReference(BaseModel): item_reference: str - """A reference to a variable in the "item" namespace. Ie, "item.name" """ + """A reference to a variable in the `item` namespace. Ie, "item.input_trajectory" """ type: Literal["item_reference"] """The type of input messages. Always `item_reference`.""" @@ -153,12 +153,18 @@ class SamplingParams(BaseModel): class CreateEvalCompletionsRunDataSource(BaseModel): source: Source - """A StoredCompletionsRunDataSource configuration describing a set of filters""" + """Determines what populates the `item` namespace in this run's data source.""" type: Literal["completions"] """The type of run data source. Always `completions`.""" input_messages: Optional[InputMessages] = None + """Used when sampling from a model. + + Dictates the structure of the messages passed into the model. Can either be a + reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template + with variable references to the `item` namespace. + """ model: Optional[str] = None """The name of the model to use for generating completions (e.g. "o3-mini").""" diff --git a/src/openai/types/evals/create_eval_completions_run_data_source_param.py b/src/openai/types/evals/create_eval_completions_run_data_source_param.py index c53064ee27..3fa4c19ad2 100644 --- a/src/openai/types/evals/create_eval_completions_run_data_source_param.py +++ b/src/openai/types/evals/create_eval_completions_run_data_source_param.py @@ -113,7 +113,7 @@ class InputMessagesTemplate(TypedDict, total=False): template: Required[Iterable[InputMessagesTemplateTemplate]] """A list of chat messages forming the prompt or context. - May include variable references to the "item" namespace, ie {{item.name}}. + May include variable references to the `item` namespace, ie {{item.name}}. """ type: Required[Literal["template"]] @@ -122,7 +122,7 @@ class InputMessagesTemplate(TypedDict, total=False): class InputMessagesItemReference(TypedDict, total=False): item_reference: Required[str] - """A reference to a variable in the "item" namespace. Ie, "item.name" """ + """A reference to a variable in the `item` namespace. Ie, "item.input_trajectory" """ type: Required[Literal["item_reference"]] """The type of input messages. Always `item_reference`.""" @@ -147,12 +147,18 @@ class SamplingParams(TypedDict, total=False): class CreateEvalCompletionsRunDataSourceParam(TypedDict, total=False): source: Required[Source] - """A StoredCompletionsRunDataSource configuration describing a set of filters""" + """Determines what populates the `item` namespace in this run's data source.""" type: Required[Literal["completions"]] """The type of run data source. Always `completions`.""" input_messages: InputMessages + """Used when sampling from a model. + + Dictates the structure of the messages passed into the model. Can either be a + reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template + with variable references to the `item` namespace. + """ model: str """The name of the model to use for generating completions (e.g. "o3-mini").""" diff --git a/src/openai/types/evals/create_eval_jsonl_run_data_source.py b/src/openai/types/evals/create_eval_jsonl_run_data_source.py index d2be56243b..ae36f8c55f 100644 --- a/src/openai/types/evals/create_eval_jsonl_run_data_source.py +++ b/src/openai/types/evals/create_eval_jsonl_run_data_source.py @@ -36,6 +36,7 @@ class SourceFileID(BaseModel): class CreateEvalJSONLRunDataSource(BaseModel): source: Source + """Determines what populates the `item` namespace in the data source.""" type: Literal["jsonl"] """The type of data source. Always `jsonl`.""" diff --git a/src/openai/types/evals/create_eval_jsonl_run_data_source_param.py b/src/openai/types/evals/create_eval_jsonl_run_data_source_param.py index b8ba48a666..217ee36346 100644 --- a/src/openai/types/evals/create_eval_jsonl_run_data_source_param.py +++ b/src/openai/types/evals/create_eval_jsonl_run_data_source_param.py @@ -41,6 +41,7 @@ class SourceFileID(TypedDict, total=False): class CreateEvalJSONLRunDataSourceParam(TypedDict, total=False): source: Required[Source] + """Determines what populates the `item` namespace in the data source.""" type: Required[Literal["jsonl"]] """The type of data source. Always `jsonl`.""" diff --git a/src/openai/types/evals/run_cancel_response.py b/src/openai/types/evals/run_cancel_response.py index eb6d689fc3..d3416129af 100644 --- a/src/openai/types/evals/run_cancel_response.py +++ b/src/openai/types/evals/run_cancel_response.py @@ -17,42 +17,42 @@ __all__ = [ "RunCancelResponse", "DataSource", - "DataSourceCompletions", - "DataSourceCompletionsSource", - "DataSourceCompletionsSourceFileContent", - "DataSourceCompletionsSourceFileContentContent", - "DataSourceCompletionsSourceFileID", - "DataSourceCompletionsSourceResponses", - "DataSourceCompletionsInputMessages", - "DataSourceCompletionsInputMessagesTemplate", - "DataSourceCompletionsInputMessagesTemplateTemplate", - "DataSourceCompletionsInputMessagesTemplateTemplateChatMessage", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItem", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText", - "DataSourceCompletionsInputMessagesItemReference", - "DataSourceCompletionsSamplingParams", + "DataSourceResponses", + "DataSourceResponsesSource", + "DataSourceResponsesSourceFileContent", + "DataSourceResponsesSourceFileContentContent", + "DataSourceResponsesSourceFileID", + "DataSourceResponsesSourceResponses", + "DataSourceResponsesInputMessages", + "DataSourceResponsesInputMessagesTemplate", + "DataSourceResponsesInputMessagesTemplateTemplate", + "DataSourceResponsesInputMessagesTemplateTemplateChatMessage", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItem", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText", + "DataSourceResponsesInputMessagesItemReference", + "DataSourceResponsesSamplingParams", "PerModelUsage", "PerTestingCriteriaResult", "ResultCounts", ] -class DataSourceCompletionsSourceFileContentContent(BaseModel): +class DataSourceResponsesSourceFileContentContent(BaseModel): item: Dict[str, object] sample: Optional[Dict[str, object]] = None -class DataSourceCompletionsSourceFileContent(BaseModel): - content: List[DataSourceCompletionsSourceFileContentContent] +class DataSourceResponsesSourceFileContent(BaseModel): + content: List[DataSourceResponsesSourceFileContentContent] """The content of the jsonl file.""" type: Literal["file_content"] """The type of jsonl source. Always `file_content`.""" -class DataSourceCompletionsSourceFileID(BaseModel): +class DataSourceResponsesSourceFileID(BaseModel): id: str """The identifier of the file.""" @@ -60,16 +60,10 @@ class DataSourceCompletionsSourceFileID(BaseModel): """The type of jsonl source. Always `file_id`.""" -class DataSourceCompletionsSourceResponses(BaseModel): +class DataSourceResponsesSourceResponses(BaseModel): type: Literal["responses"] """The type of run data source. Always `responses`.""" - allow_parallel_tool_calls: Optional[bool] = None - """Whether to allow parallel tool calls. - - This is a query parameter used to select responses. - """ - created_after: Optional[int] = None """Only include items created after this timestamp (inclusive). @@ -82,14 +76,8 @@ class DataSourceCompletionsSourceResponses(BaseModel): This is a query parameter used to select responses. """ - has_tool_calls: Optional[bool] = None - """Whether the response has tool calls. - - This is a query parameter used to select responses. - """ - instructions_search: Optional[str] = None - """Optional search string for instructions. + """Optional string to search the 'instructions' field. This is a query parameter used to select responses. """ @@ -115,6 +103,9 @@ class DataSourceCompletionsSourceResponses(BaseModel): temperature: Optional[float] = None """Sampling temperature. This is a query parameter used to select responses.""" + tools: Optional[List[str]] = None + """List of tool names. This is a query parameter used to select responses.""" + top_p: Optional[float] = None """Nucleus sampling parameter. This is a query parameter used to select responses.""" @@ -122,15 +113,13 @@ class DataSourceCompletionsSourceResponses(BaseModel): """List of user identifiers. This is a query parameter used to select responses.""" -DataSourceCompletionsSource: TypeAlias = Annotated[ - Union[ - DataSourceCompletionsSourceFileContent, DataSourceCompletionsSourceFileID, DataSourceCompletionsSourceResponses - ], +DataSourceResponsesSource: TypeAlias = Annotated[ + Union[DataSourceResponsesSourceFileContent, DataSourceResponsesSourceFileID, DataSourceResponsesSourceResponses], PropertyInfo(discriminator="type"), ] -class DataSourceCompletionsInputMessagesTemplateTemplateChatMessage(BaseModel): +class DataSourceResponsesInputMessagesTemplateTemplateChatMessage(BaseModel): content: str """The content of the message.""" @@ -138,7 +127,7 @@ class DataSourceCompletionsInputMessagesTemplateTemplateChatMessage(BaseModel): """The role of the message (e.g. "system", "assistant", "user").""" -class DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText(BaseModel): +class DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText(BaseModel): text: str """The text output from the model.""" @@ -146,13 +135,13 @@ class DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputTex """The type of the output text. Always `output_text`.""" -DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent: TypeAlias = Union[ - str, ResponseInputText, DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText +DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent: TypeAlias = Union[ + str, ResponseInputText, DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText ] -class DataSourceCompletionsInputMessagesTemplateTemplateEvalItem(BaseModel): - content: DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent +class DataSourceResponsesInputMessagesTemplateTemplateEvalItem(BaseModel): + content: DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent """Text inputs to the model - can contain template strings.""" role: Literal["user", "assistant", "system", "developer"] @@ -165,38 +154,38 @@ class DataSourceCompletionsInputMessagesTemplateTemplateEvalItem(BaseModel): """The type of the message input. Always `message`.""" -DataSourceCompletionsInputMessagesTemplateTemplate: TypeAlias = Union[ - DataSourceCompletionsInputMessagesTemplateTemplateChatMessage, - DataSourceCompletionsInputMessagesTemplateTemplateEvalItem, +DataSourceResponsesInputMessagesTemplateTemplate: TypeAlias = Union[ + DataSourceResponsesInputMessagesTemplateTemplateChatMessage, + DataSourceResponsesInputMessagesTemplateTemplateEvalItem, ] -class DataSourceCompletionsInputMessagesTemplate(BaseModel): - template: List[DataSourceCompletionsInputMessagesTemplateTemplate] +class DataSourceResponsesInputMessagesTemplate(BaseModel): + template: List[DataSourceResponsesInputMessagesTemplateTemplate] """A list of chat messages forming the prompt or context. - May include variable references to the "item" namespace, ie {{item.name}}. + May include variable references to the `item` namespace, ie {{item.name}}. """ type: Literal["template"] """The type of input messages. Always `template`.""" -class DataSourceCompletionsInputMessagesItemReference(BaseModel): +class DataSourceResponsesInputMessagesItemReference(BaseModel): item_reference: str - """A reference to a variable in the "item" namespace. Ie, "item.name" """ + """A reference to a variable in the `item` namespace. Ie, "item.name" """ type: Literal["item_reference"] """The type of input messages. Always `item_reference`.""" -DataSourceCompletionsInputMessages: TypeAlias = Annotated[ - Union[DataSourceCompletionsInputMessagesTemplate, DataSourceCompletionsInputMessagesItemReference], +DataSourceResponsesInputMessages: TypeAlias = Annotated[ + Union[DataSourceResponsesInputMessagesTemplate, DataSourceResponsesInputMessagesItemReference], PropertyInfo(discriminator="type"), ] -class DataSourceCompletionsSamplingParams(BaseModel): +class DataSourceResponsesSamplingParams(BaseModel): max_completion_tokens: Optional[int] = None """The maximum number of tokens in the generated output.""" @@ -210,23 +199,29 @@ class DataSourceCompletionsSamplingParams(BaseModel): """An alternative to temperature for nucleus sampling; 1.0 includes all tokens.""" -class DataSourceCompletions(BaseModel): - source: DataSourceCompletionsSource - """A EvalResponsesSource object describing a run data source configuration.""" +class DataSourceResponses(BaseModel): + source: DataSourceResponsesSource + """Determines what populates the `item` namespace in this run's data source.""" + + type: Literal["responses"] + """The type of run data source. Always `responses`.""" - type: Literal["completions"] - """The type of run data source. Always `completions`.""" + input_messages: Optional[DataSourceResponsesInputMessages] = None + """Used when sampling from a model. - input_messages: Optional[DataSourceCompletionsInputMessages] = None + Dictates the structure of the messages passed into the model. Can either be a + reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template + with variable references to the `item` namespace. + """ model: Optional[str] = None """The name of the model to use for generating completions (e.g. "o3-mini").""" - sampling_params: Optional[DataSourceCompletionsSamplingParams] = None + sampling_params: Optional[DataSourceResponsesSamplingParams] = None DataSource: TypeAlias = Annotated[ - Union[CreateEvalJSONLRunDataSource, CreateEvalCompletionsRunDataSource, DataSourceCompletions], + Union[CreateEvalJSONLRunDataSource, CreateEvalCompletionsRunDataSource, DataSourceResponses], PropertyInfo(discriminator="type"), ] diff --git a/src/openai/types/evals/run_create_params.py b/src/openai/types/evals/run_create_params.py index 0c9720ea7a..5aa2398f36 100644 --- a/src/openai/types/evals/run_create_params.py +++ b/src/openai/types/evals/run_create_params.py @@ -76,12 +76,6 @@ class DataSourceCreateEvalResponsesRunDataSourceSourceResponses(TypedDict, total type: Required[Literal["responses"]] """The type of run data source. Always `responses`.""" - allow_parallel_tool_calls: Optional[bool] - """Whether to allow parallel tool calls. - - This is a query parameter used to select responses. - """ - created_after: Optional[int] """Only include items created after this timestamp (inclusive). @@ -94,14 +88,8 @@ class DataSourceCreateEvalResponsesRunDataSourceSourceResponses(TypedDict, total This is a query parameter used to select responses. """ - has_tool_calls: Optional[bool] - """Whether the response has tool calls. - - This is a query parameter used to select responses. - """ - instructions_search: Optional[str] - """Optional search string for instructions. + """Optional string to search the 'instructions' field. This is a query parameter used to select responses. """ @@ -127,6 +115,9 @@ class DataSourceCreateEvalResponsesRunDataSourceSourceResponses(TypedDict, total temperature: Optional[float] """Sampling temperature. This is a query parameter used to select responses.""" + tools: Optional[List[str]] + """List of tool names. This is a query parameter used to select responses.""" + top_p: Optional[float] """Nucleus sampling parameter. This is a query parameter used to select responses.""" @@ -190,7 +181,7 @@ class DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplate(TypedDict, template: Required[Iterable[DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplate]] """A list of chat messages forming the prompt or context. - May include variable references to the "item" namespace, ie {{item.name}}. + May include variable references to the `item` namespace, ie {{item.name}}. """ type: Required[Literal["template"]] @@ -199,7 +190,7 @@ class DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplate(TypedDict, class DataSourceCreateEvalResponsesRunDataSourceInputMessagesItemReference(TypedDict, total=False): item_reference: Required[str] - """A reference to a variable in the "item" namespace. Ie, "item.name" """ + """A reference to a variable in the `item` namespace. Ie, "item.name" """ type: Required[Literal["item_reference"]] """The type of input messages. Always `item_reference`.""" @@ -227,12 +218,18 @@ class DataSourceCreateEvalResponsesRunDataSourceSamplingParams(TypedDict, total= class DataSourceCreateEvalResponsesRunDataSource(TypedDict, total=False): source: Required[DataSourceCreateEvalResponsesRunDataSourceSource] - """A EvalResponsesSource object describing a run data source configuration.""" + """Determines what populates the `item` namespace in this run's data source.""" - type: Required[Literal["completions"]] - """The type of run data source. Always `completions`.""" + type: Required[Literal["responses"]] + """The type of run data source. Always `responses`.""" input_messages: DataSourceCreateEvalResponsesRunDataSourceInputMessages + """Used when sampling from a model. + + Dictates the structure of the messages passed into the model. Can either be a + reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template + with variable references to the `item` namespace. + """ model: str """The name of the model to use for generating completions (e.g. "o3-mini").""" diff --git a/src/openai/types/evals/run_create_response.py b/src/openai/types/evals/run_create_response.py index 459399511c..51aed2080f 100644 --- a/src/openai/types/evals/run_create_response.py +++ b/src/openai/types/evals/run_create_response.py @@ -17,42 +17,42 @@ __all__ = [ "RunCreateResponse", "DataSource", - "DataSourceCompletions", - "DataSourceCompletionsSource", - "DataSourceCompletionsSourceFileContent", - "DataSourceCompletionsSourceFileContentContent", - "DataSourceCompletionsSourceFileID", - "DataSourceCompletionsSourceResponses", - "DataSourceCompletionsInputMessages", - "DataSourceCompletionsInputMessagesTemplate", - "DataSourceCompletionsInputMessagesTemplateTemplate", - "DataSourceCompletionsInputMessagesTemplateTemplateChatMessage", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItem", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText", - "DataSourceCompletionsInputMessagesItemReference", - "DataSourceCompletionsSamplingParams", + "DataSourceResponses", + "DataSourceResponsesSource", + "DataSourceResponsesSourceFileContent", + "DataSourceResponsesSourceFileContentContent", + "DataSourceResponsesSourceFileID", + "DataSourceResponsesSourceResponses", + "DataSourceResponsesInputMessages", + "DataSourceResponsesInputMessagesTemplate", + "DataSourceResponsesInputMessagesTemplateTemplate", + "DataSourceResponsesInputMessagesTemplateTemplateChatMessage", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItem", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText", + "DataSourceResponsesInputMessagesItemReference", + "DataSourceResponsesSamplingParams", "PerModelUsage", "PerTestingCriteriaResult", "ResultCounts", ] -class DataSourceCompletionsSourceFileContentContent(BaseModel): +class DataSourceResponsesSourceFileContentContent(BaseModel): item: Dict[str, object] sample: Optional[Dict[str, object]] = None -class DataSourceCompletionsSourceFileContent(BaseModel): - content: List[DataSourceCompletionsSourceFileContentContent] +class DataSourceResponsesSourceFileContent(BaseModel): + content: List[DataSourceResponsesSourceFileContentContent] """The content of the jsonl file.""" type: Literal["file_content"] """The type of jsonl source. Always `file_content`.""" -class DataSourceCompletionsSourceFileID(BaseModel): +class DataSourceResponsesSourceFileID(BaseModel): id: str """The identifier of the file.""" @@ -60,16 +60,10 @@ class DataSourceCompletionsSourceFileID(BaseModel): """The type of jsonl source. Always `file_id`.""" -class DataSourceCompletionsSourceResponses(BaseModel): +class DataSourceResponsesSourceResponses(BaseModel): type: Literal["responses"] """The type of run data source. Always `responses`.""" - allow_parallel_tool_calls: Optional[bool] = None - """Whether to allow parallel tool calls. - - This is a query parameter used to select responses. - """ - created_after: Optional[int] = None """Only include items created after this timestamp (inclusive). @@ -82,14 +76,8 @@ class DataSourceCompletionsSourceResponses(BaseModel): This is a query parameter used to select responses. """ - has_tool_calls: Optional[bool] = None - """Whether the response has tool calls. - - This is a query parameter used to select responses. - """ - instructions_search: Optional[str] = None - """Optional search string for instructions. + """Optional string to search the 'instructions' field. This is a query parameter used to select responses. """ @@ -115,6 +103,9 @@ class DataSourceCompletionsSourceResponses(BaseModel): temperature: Optional[float] = None """Sampling temperature. This is a query parameter used to select responses.""" + tools: Optional[List[str]] = None + """List of tool names. This is a query parameter used to select responses.""" + top_p: Optional[float] = None """Nucleus sampling parameter. This is a query parameter used to select responses.""" @@ -122,15 +113,13 @@ class DataSourceCompletionsSourceResponses(BaseModel): """List of user identifiers. This is a query parameter used to select responses.""" -DataSourceCompletionsSource: TypeAlias = Annotated[ - Union[ - DataSourceCompletionsSourceFileContent, DataSourceCompletionsSourceFileID, DataSourceCompletionsSourceResponses - ], +DataSourceResponsesSource: TypeAlias = Annotated[ + Union[DataSourceResponsesSourceFileContent, DataSourceResponsesSourceFileID, DataSourceResponsesSourceResponses], PropertyInfo(discriminator="type"), ] -class DataSourceCompletionsInputMessagesTemplateTemplateChatMessage(BaseModel): +class DataSourceResponsesInputMessagesTemplateTemplateChatMessage(BaseModel): content: str """The content of the message.""" @@ -138,7 +127,7 @@ class DataSourceCompletionsInputMessagesTemplateTemplateChatMessage(BaseModel): """The role of the message (e.g. "system", "assistant", "user").""" -class DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText(BaseModel): +class DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText(BaseModel): text: str """The text output from the model.""" @@ -146,13 +135,13 @@ class DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputTex """The type of the output text. Always `output_text`.""" -DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent: TypeAlias = Union[ - str, ResponseInputText, DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText +DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent: TypeAlias = Union[ + str, ResponseInputText, DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText ] -class DataSourceCompletionsInputMessagesTemplateTemplateEvalItem(BaseModel): - content: DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent +class DataSourceResponsesInputMessagesTemplateTemplateEvalItem(BaseModel): + content: DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent """Text inputs to the model - can contain template strings.""" role: Literal["user", "assistant", "system", "developer"] @@ -165,38 +154,38 @@ class DataSourceCompletionsInputMessagesTemplateTemplateEvalItem(BaseModel): """The type of the message input. Always `message`.""" -DataSourceCompletionsInputMessagesTemplateTemplate: TypeAlias = Union[ - DataSourceCompletionsInputMessagesTemplateTemplateChatMessage, - DataSourceCompletionsInputMessagesTemplateTemplateEvalItem, +DataSourceResponsesInputMessagesTemplateTemplate: TypeAlias = Union[ + DataSourceResponsesInputMessagesTemplateTemplateChatMessage, + DataSourceResponsesInputMessagesTemplateTemplateEvalItem, ] -class DataSourceCompletionsInputMessagesTemplate(BaseModel): - template: List[DataSourceCompletionsInputMessagesTemplateTemplate] +class DataSourceResponsesInputMessagesTemplate(BaseModel): + template: List[DataSourceResponsesInputMessagesTemplateTemplate] """A list of chat messages forming the prompt or context. - May include variable references to the "item" namespace, ie {{item.name}}. + May include variable references to the `item` namespace, ie {{item.name}}. """ type: Literal["template"] """The type of input messages. Always `template`.""" -class DataSourceCompletionsInputMessagesItemReference(BaseModel): +class DataSourceResponsesInputMessagesItemReference(BaseModel): item_reference: str - """A reference to a variable in the "item" namespace. Ie, "item.name" """ + """A reference to a variable in the `item` namespace. Ie, "item.name" """ type: Literal["item_reference"] """The type of input messages. Always `item_reference`.""" -DataSourceCompletionsInputMessages: TypeAlias = Annotated[ - Union[DataSourceCompletionsInputMessagesTemplate, DataSourceCompletionsInputMessagesItemReference], +DataSourceResponsesInputMessages: TypeAlias = Annotated[ + Union[DataSourceResponsesInputMessagesTemplate, DataSourceResponsesInputMessagesItemReference], PropertyInfo(discriminator="type"), ] -class DataSourceCompletionsSamplingParams(BaseModel): +class DataSourceResponsesSamplingParams(BaseModel): max_completion_tokens: Optional[int] = None """The maximum number of tokens in the generated output.""" @@ -210,23 +199,29 @@ class DataSourceCompletionsSamplingParams(BaseModel): """An alternative to temperature for nucleus sampling; 1.0 includes all tokens.""" -class DataSourceCompletions(BaseModel): - source: DataSourceCompletionsSource - """A EvalResponsesSource object describing a run data source configuration.""" +class DataSourceResponses(BaseModel): + source: DataSourceResponsesSource + """Determines what populates the `item` namespace in this run's data source.""" + + type: Literal["responses"] + """The type of run data source. Always `responses`.""" - type: Literal["completions"] - """The type of run data source. Always `completions`.""" + input_messages: Optional[DataSourceResponsesInputMessages] = None + """Used when sampling from a model. - input_messages: Optional[DataSourceCompletionsInputMessages] = None + Dictates the structure of the messages passed into the model. Can either be a + reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template + with variable references to the `item` namespace. + """ model: Optional[str] = None """The name of the model to use for generating completions (e.g. "o3-mini").""" - sampling_params: Optional[DataSourceCompletionsSamplingParams] = None + sampling_params: Optional[DataSourceResponsesSamplingParams] = None DataSource: TypeAlias = Annotated[ - Union[CreateEvalJSONLRunDataSource, CreateEvalCompletionsRunDataSource, DataSourceCompletions], + Union[CreateEvalJSONLRunDataSource, CreateEvalCompletionsRunDataSource, DataSourceResponses], PropertyInfo(discriminator="type"), ] diff --git a/src/openai/types/evals/run_list_response.py b/src/openai/types/evals/run_list_response.py index 278ceeabed..f1d0b01da9 100644 --- a/src/openai/types/evals/run_list_response.py +++ b/src/openai/types/evals/run_list_response.py @@ -17,42 +17,42 @@ __all__ = [ "RunListResponse", "DataSource", - "DataSourceCompletions", - "DataSourceCompletionsSource", - "DataSourceCompletionsSourceFileContent", - "DataSourceCompletionsSourceFileContentContent", - "DataSourceCompletionsSourceFileID", - "DataSourceCompletionsSourceResponses", - "DataSourceCompletionsInputMessages", - "DataSourceCompletionsInputMessagesTemplate", - "DataSourceCompletionsInputMessagesTemplateTemplate", - "DataSourceCompletionsInputMessagesTemplateTemplateChatMessage", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItem", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText", - "DataSourceCompletionsInputMessagesItemReference", - "DataSourceCompletionsSamplingParams", + "DataSourceResponses", + "DataSourceResponsesSource", + "DataSourceResponsesSourceFileContent", + "DataSourceResponsesSourceFileContentContent", + "DataSourceResponsesSourceFileID", + "DataSourceResponsesSourceResponses", + "DataSourceResponsesInputMessages", + "DataSourceResponsesInputMessagesTemplate", + "DataSourceResponsesInputMessagesTemplateTemplate", + "DataSourceResponsesInputMessagesTemplateTemplateChatMessage", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItem", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText", + "DataSourceResponsesInputMessagesItemReference", + "DataSourceResponsesSamplingParams", "PerModelUsage", "PerTestingCriteriaResult", "ResultCounts", ] -class DataSourceCompletionsSourceFileContentContent(BaseModel): +class DataSourceResponsesSourceFileContentContent(BaseModel): item: Dict[str, object] sample: Optional[Dict[str, object]] = None -class DataSourceCompletionsSourceFileContent(BaseModel): - content: List[DataSourceCompletionsSourceFileContentContent] +class DataSourceResponsesSourceFileContent(BaseModel): + content: List[DataSourceResponsesSourceFileContentContent] """The content of the jsonl file.""" type: Literal["file_content"] """The type of jsonl source. Always `file_content`.""" -class DataSourceCompletionsSourceFileID(BaseModel): +class DataSourceResponsesSourceFileID(BaseModel): id: str """The identifier of the file.""" @@ -60,16 +60,10 @@ class DataSourceCompletionsSourceFileID(BaseModel): """The type of jsonl source. Always `file_id`.""" -class DataSourceCompletionsSourceResponses(BaseModel): +class DataSourceResponsesSourceResponses(BaseModel): type: Literal["responses"] """The type of run data source. Always `responses`.""" - allow_parallel_tool_calls: Optional[bool] = None - """Whether to allow parallel tool calls. - - This is a query parameter used to select responses. - """ - created_after: Optional[int] = None """Only include items created after this timestamp (inclusive). @@ -82,14 +76,8 @@ class DataSourceCompletionsSourceResponses(BaseModel): This is a query parameter used to select responses. """ - has_tool_calls: Optional[bool] = None - """Whether the response has tool calls. - - This is a query parameter used to select responses. - """ - instructions_search: Optional[str] = None - """Optional search string for instructions. + """Optional string to search the 'instructions' field. This is a query parameter used to select responses. """ @@ -115,6 +103,9 @@ class DataSourceCompletionsSourceResponses(BaseModel): temperature: Optional[float] = None """Sampling temperature. This is a query parameter used to select responses.""" + tools: Optional[List[str]] = None + """List of tool names. This is a query parameter used to select responses.""" + top_p: Optional[float] = None """Nucleus sampling parameter. This is a query parameter used to select responses.""" @@ -122,15 +113,13 @@ class DataSourceCompletionsSourceResponses(BaseModel): """List of user identifiers. This is a query parameter used to select responses.""" -DataSourceCompletionsSource: TypeAlias = Annotated[ - Union[ - DataSourceCompletionsSourceFileContent, DataSourceCompletionsSourceFileID, DataSourceCompletionsSourceResponses - ], +DataSourceResponsesSource: TypeAlias = Annotated[ + Union[DataSourceResponsesSourceFileContent, DataSourceResponsesSourceFileID, DataSourceResponsesSourceResponses], PropertyInfo(discriminator="type"), ] -class DataSourceCompletionsInputMessagesTemplateTemplateChatMessage(BaseModel): +class DataSourceResponsesInputMessagesTemplateTemplateChatMessage(BaseModel): content: str """The content of the message.""" @@ -138,7 +127,7 @@ class DataSourceCompletionsInputMessagesTemplateTemplateChatMessage(BaseModel): """The role of the message (e.g. "system", "assistant", "user").""" -class DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText(BaseModel): +class DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText(BaseModel): text: str """The text output from the model.""" @@ -146,13 +135,13 @@ class DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputTex """The type of the output text. Always `output_text`.""" -DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent: TypeAlias = Union[ - str, ResponseInputText, DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText +DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent: TypeAlias = Union[ + str, ResponseInputText, DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText ] -class DataSourceCompletionsInputMessagesTemplateTemplateEvalItem(BaseModel): - content: DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent +class DataSourceResponsesInputMessagesTemplateTemplateEvalItem(BaseModel): + content: DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent """Text inputs to the model - can contain template strings.""" role: Literal["user", "assistant", "system", "developer"] @@ -165,38 +154,38 @@ class DataSourceCompletionsInputMessagesTemplateTemplateEvalItem(BaseModel): """The type of the message input. Always `message`.""" -DataSourceCompletionsInputMessagesTemplateTemplate: TypeAlias = Union[ - DataSourceCompletionsInputMessagesTemplateTemplateChatMessage, - DataSourceCompletionsInputMessagesTemplateTemplateEvalItem, +DataSourceResponsesInputMessagesTemplateTemplate: TypeAlias = Union[ + DataSourceResponsesInputMessagesTemplateTemplateChatMessage, + DataSourceResponsesInputMessagesTemplateTemplateEvalItem, ] -class DataSourceCompletionsInputMessagesTemplate(BaseModel): - template: List[DataSourceCompletionsInputMessagesTemplateTemplate] +class DataSourceResponsesInputMessagesTemplate(BaseModel): + template: List[DataSourceResponsesInputMessagesTemplateTemplate] """A list of chat messages forming the prompt or context. - May include variable references to the "item" namespace, ie {{item.name}}. + May include variable references to the `item` namespace, ie {{item.name}}. """ type: Literal["template"] """The type of input messages. Always `template`.""" -class DataSourceCompletionsInputMessagesItemReference(BaseModel): +class DataSourceResponsesInputMessagesItemReference(BaseModel): item_reference: str - """A reference to a variable in the "item" namespace. Ie, "item.name" """ + """A reference to a variable in the `item` namespace. Ie, "item.name" """ type: Literal["item_reference"] """The type of input messages. Always `item_reference`.""" -DataSourceCompletionsInputMessages: TypeAlias = Annotated[ - Union[DataSourceCompletionsInputMessagesTemplate, DataSourceCompletionsInputMessagesItemReference], +DataSourceResponsesInputMessages: TypeAlias = Annotated[ + Union[DataSourceResponsesInputMessagesTemplate, DataSourceResponsesInputMessagesItemReference], PropertyInfo(discriminator="type"), ] -class DataSourceCompletionsSamplingParams(BaseModel): +class DataSourceResponsesSamplingParams(BaseModel): max_completion_tokens: Optional[int] = None """The maximum number of tokens in the generated output.""" @@ -210,23 +199,29 @@ class DataSourceCompletionsSamplingParams(BaseModel): """An alternative to temperature for nucleus sampling; 1.0 includes all tokens.""" -class DataSourceCompletions(BaseModel): - source: DataSourceCompletionsSource - """A EvalResponsesSource object describing a run data source configuration.""" +class DataSourceResponses(BaseModel): + source: DataSourceResponsesSource + """Determines what populates the `item` namespace in this run's data source.""" + + type: Literal["responses"] + """The type of run data source. Always `responses`.""" - type: Literal["completions"] - """The type of run data source. Always `completions`.""" + input_messages: Optional[DataSourceResponsesInputMessages] = None + """Used when sampling from a model. - input_messages: Optional[DataSourceCompletionsInputMessages] = None + Dictates the structure of the messages passed into the model. Can either be a + reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template + with variable references to the `item` namespace. + """ model: Optional[str] = None """The name of the model to use for generating completions (e.g. "o3-mini").""" - sampling_params: Optional[DataSourceCompletionsSamplingParams] = None + sampling_params: Optional[DataSourceResponsesSamplingParams] = None DataSource: TypeAlias = Annotated[ - Union[CreateEvalJSONLRunDataSource, CreateEvalCompletionsRunDataSource, DataSourceCompletions], + Union[CreateEvalJSONLRunDataSource, CreateEvalCompletionsRunDataSource, DataSourceResponses], PropertyInfo(discriminator="type"), ] diff --git a/src/openai/types/evals/run_retrieve_response.py b/src/openai/types/evals/run_retrieve_response.py index e142f31b14..6c5951b4eb 100644 --- a/src/openai/types/evals/run_retrieve_response.py +++ b/src/openai/types/evals/run_retrieve_response.py @@ -17,42 +17,42 @@ __all__ = [ "RunRetrieveResponse", "DataSource", - "DataSourceCompletions", - "DataSourceCompletionsSource", - "DataSourceCompletionsSourceFileContent", - "DataSourceCompletionsSourceFileContentContent", - "DataSourceCompletionsSourceFileID", - "DataSourceCompletionsSourceResponses", - "DataSourceCompletionsInputMessages", - "DataSourceCompletionsInputMessagesTemplate", - "DataSourceCompletionsInputMessagesTemplateTemplate", - "DataSourceCompletionsInputMessagesTemplateTemplateChatMessage", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItem", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent", - "DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText", - "DataSourceCompletionsInputMessagesItemReference", - "DataSourceCompletionsSamplingParams", + "DataSourceResponses", + "DataSourceResponsesSource", + "DataSourceResponsesSourceFileContent", + "DataSourceResponsesSourceFileContentContent", + "DataSourceResponsesSourceFileID", + "DataSourceResponsesSourceResponses", + "DataSourceResponsesInputMessages", + "DataSourceResponsesInputMessagesTemplate", + "DataSourceResponsesInputMessagesTemplateTemplate", + "DataSourceResponsesInputMessagesTemplateTemplateChatMessage", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItem", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent", + "DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText", + "DataSourceResponsesInputMessagesItemReference", + "DataSourceResponsesSamplingParams", "PerModelUsage", "PerTestingCriteriaResult", "ResultCounts", ] -class DataSourceCompletionsSourceFileContentContent(BaseModel): +class DataSourceResponsesSourceFileContentContent(BaseModel): item: Dict[str, object] sample: Optional[Dict[str, object]] = None -class DataSourceCompletionsSourceFileContent(BaseModel): - content: List[DataSourceCompletionsSourceFileContentContent] +class DataSourceResponsesSourceFileContent(BaseModel): + content: List[DataSourceResponsesSourceFileContentContent] """The content of the jsonl file.""" type: Literal["file_content"] """The type of jsonl source. Always `file_content`.""" -class DataSourceCompletionsSourceFileID(BaseModel): +class DataSourceResponsesSourceFileID(BaseModel): id: str """The identifier of the file.""" @@ -60,16 +60,10 @@ class DataSourceCompletionsSourceFileID(BaseModel): """The type of jsonl source. Always `file_id`.""" -class DataSourceCompletionsSourceResponses(BaseModel): +class DataSourceResponsesSourceResponses(BaseModel): type: Literal["responses"] """The type of run data source. Always `responses`.""" - allow_parallel_tool_calls: Optional[bool] = None - """Whether to allow parallel tool calls. - - This is a query parameter used to select responses. - """ - created_after: Optional[int] = None """Only include items created after this timestamp (inclusive). @@ -82,14 +76,8 @@ class DataSourceCompletionsSourceResponses(BaseModel): This is a query parameter used to select responses. """ - has_tool_calls: Optional[bool] = None - """Whether the response has tool calls. - - This is a query parameter used to select responses. - """ - instructions_search: Optional[str] = None - """Optional search string for instructions. + """Optional string to search the 'instructions' field. This is a query parameter used to select responses. """ @@ -115,6 +103,9 @@ class DataSourceCompletionsSourceResponses(BaseModel): temperature: Optional[float] = None """Sampling temperature. This is a query parameter used to select responses.""" + tools: Optional[List[str]] = None + """List of tool names. This is a query parameter used to select responses.""" + top_p: Optional[float] = None """Nucleus sampling parameter. This is a query parameter used to select responses.""" @@ -122,15 +113,13 @@ class DataSourceCompletionsSourceResponses(BaseModel): """List of user identifiers. This is a query parameter used to select responses.""" -DataSourceCompletionsSource: TypeAlias = Annotated[ - Union[ - DataSourceCompletionsSourceFileContent, DataSourceCompletionsSourceFileID, DataSourceCompletionsSourceResponses - ], +DataSourceResponsesSource: TypeAlias = Annotated[ + Union[DataSourceResponsesSourceFileContent, DataSourceResponsesSourceFileID, DataSourceResponsesSourceResponses], PropertyInfo(discriminator="type"), ] -class DataSourceCompletionsInputMessagesTemplateTemplateChatMessage(BaseModel): +class DataSourceResponsesInputMessagesTemplateTemplateChatMessage(BaseModel): content: str """The content of the message.""" @@ -138,7 +127,7 @@ class DataSourceCompletionsInputMessagesTemplateTemplateChatMessage(BaseModel): """The role of the message (e.g. "system", "assistant", "user").""" -class DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText(BaseModel): +class DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText(BaseModel): text: str """The text output from the model.""" @@ -146,13 +135,13 @@ class DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputTex """The type of the output text. Always `output_text`.""" -DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent: TypeAlias = Union[ - str, ResponseInputText, DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContentOutputText +DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent: TypeAlias = Union[ + str, ResponseInputText, DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText ] -class DataSourceCompletionsInputMessagesTemplateTemplateEvalItem(BaseModel): - content: DataSourceCompletionsInputMessagesTemplateTemplateEvalItemContent +class DataSourceResponsesInputMessagesTemplateTemplateEvalItem(BaseModel): + content: DataSourceResponsesInputMessagesTemplateTemplateEvalItemContent """Text inputs to the model - can contain template strings.""" role: Literal["user", "assistant", "system", "developer"] @@ -165,38 +154,38 @@ class DataSourceCompletionsInputMessagesTemplateTemplateEvalItem(BaseModel): """The type of the message input. Always `message`.""" -DataSourceCompletionsInputMessagesTemplateTemplate: TypeAlias = Union[ - DataSourceCompletionsInputMessagesTemplateTemplateChatMessage, - DataSourceCompletionsInputMessagesTemplateTemplateEvalItem, +DataSourceResponsesInputMessagesTemplateTemplate: TypeAlias = Union[ + DataSourceResponsesInputMessagesTemplateTemplateChatMessage, + DataSourceResponsesInputMessagesTemplateTemplateEvalItem, ] -class DataSourceCompletionsInputMessagesTemplate(BaseModel): - template: List[DataSourceCompletionsInputMessagesTemplateTemplate] +class DataSourceResponsesInputMessagesTemplate(BaseModel): + template: List[DataSourceResponsesInputMessagesTemplateTemplate] """A list of chat messages forming the prompt or context. - May include variable references to the "item" namespace, ie {{item.name}}. + May include variable references to the `item` namespace, ie {{item.name}}. """ type: Literal["template"] """The type of input messages. Always `template`.""" -class DataSourceCompletionsInputMessagesItemReference(BaseModel): +class DataSourceResponsesInputMessagesItemReference(BaseModel): item_reference: str - """A reference to a variable in the "item" namespace. Ie, "item.name" """ + """A reference to a variable in the `item` namespace. Ie, "item.name" """ type: Literal["item_reference"] """The type of input messages. Always `item_reference`.""" -DataSourceCompletionsInputMessages: TypeAlias = Annotated[ - Union[DataSourceCompletionsInputMessagesTemplate, DataSourceCompletionsInputMessagesItemReference], +DataSourceResponsesInputMessages: TypeAlias = Annotated[ + Union[DataSourceResponsesInputMessagesTemplate, DataSourceResponsesInputMessagesItemReference], PropertyInfo(discriminator="type"), ] -class DataSourceCompletionsSamplingParams(BaseModel): +class DataSourceResponsesSamplingParams(BaseModel): max_completion_tokens: Optional[int] = None """The maximum number of tokens in the generated output.""" @@ -210,23 +199,29 @@ class DataSourceCompletionsSamplingParams(BaseModel): """An alternative to temperature for nucleus sampling; 1.0 includes all tokens.""" -class DataSourceCompletions(BaseModel): - source: DataSourceCompletionsSource - """A EvalResponsesSource object describing a run data source configuration.""" +class DataSourceResponses(BaseModel): + source: DataSourceResponsesSource + """Determines what populates the `item` namespace in this run's data source.""" + + type: Literal["responses"] + """The type of run data source. Always `responses`.""" - type: Literal["completions"] - """The type of run data source. Always `completions`.""" + input_messages: Optional[DataSourceResponsesInputMessages] = None + """Used when sampling from a model. - input_messages: Optional[DataSourceCompletionsInputMessages] = None + Dictates the structure of the messages passed into the model. Can either be a + reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template + with variable references to the `item` namespace. + """ model: Optional[str] = None """The name of the model to use for generating completions (e.g. "o3-mini").""" - sampling_params: Optional[DataSourceCompletionsSamplingParams] = None + sampling_params: Optional[DataSourceResponsesSamplingParams] = None DataSource: TypeAlias = Annotated[ - Union[CreateEvalJSONLRunDataSource, CreateEvalCompletionsRunDataSource, DataSourceCompletions], + Union[CreateEvalJSONLRunDataSource, CreateEvalCompletionsRunDataSource, DataSourceResponses], PropertyInfo(discriminator="type"), ] diff --git a/src/openai/types/fine_tuning/fine_tuning_job.py b/src/openai/types/fine_tuning/fine_tuning_job.py index f626fbba64..b6123f8ba6 100644 --- a/src/openai/types/fine_tuning/fine_tuning_job.py +++ b/src/openai/types/fine_tuning/fine_tuning_job.py @@ -28,7 +28,7 @@ class Error(BaseModel): class Hyperparameters(BaseModel): - batch_size: Union[Literal["auto"], int, None] = None + batch_size: Union[Literal["auto"], int, Optional[object], None] = None """Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but diff --git a/src/openai/types/graders/multi_grader.py b/src/openai/types/graders/multi_grader.py index ee9b31d2b0..220de2e61b 100644 --- a/src/openai/types/graders/multi_grader.py +++ b/src/openai/types/graders/multi_grader.py @@ -25,4 +25,4 @@ class MultiGrader(BaseModel): """The name of the grader.""" type: Literal["multi"] - """The type of grader.""" + """The object type, which is always `multi`.""" diff --git a/src/openai/types/graders/multi_grader_param.py b/src/openai/types/graders/multi_grader_param.py index 4dd1a48530..2984b5668f 100644 --- a/src/openai/types/graders/multi_grader_param.py +++ b/src/openai/types/graders/multi_grader_param.py @@ -28,4 +28,4 @@ class MultiGraderParam(TypedDict, total=False): """The name of the grader.""" type: Required[Literal["multi"]] - """The type of grader.""" + """The object type, which is always `multi`.""" diff --git a/src/openai/types/shared/chat_model.py b/src/openai/types/shared/chat_model.py index 4869cd325c..75069e7a98 100644 --- a/src/openai/types/shared/chat_model.py +++ b/src/openai/types/shared/chat_model.py @@ -37,6 +37,7 @@ "gpt-4o-search-preview-2025-03-11", "gpt-4o-mini-search-preview-2025-03-11", "chatgpt-4o-latest", + "codex-mini-latest", "gpt-4o-mini", "gpt-4o-mini-2024-07-18", "gpt-4-turbo", diff --git a/src/openai/types/shared_params/chat_model.py b/src/openai/types/shared_params/chat_model.py index 99e082fc11..c421744b8a 100644 --- a/src/openai/types/shared_params/chat_model.py +++ b/src/openai/types/shared_params/chat_model.py @@ -39,6 +39,7 @@ "gpt-4o-search-preview-2025-03-11", "gpt-4o-mini-search-preview-2025-03-11", "chatgpt-4o-latest", + "codex-mini-latest", "gpt-4o-mini", "gpt-4o-mini-2024-07-18", "gpt-4-turbo", diff --git a/tests/api_resources/audio/test_transcriptions.py b/tests/api_resources/audio/test_transcriptions.py index 19215e11df..753acdecf6 100644 --- a/tests/api_resources/audio/test_transcriptions.py +++ b/tests/api_resources/audio/test_transcriptions.py @@ -30,6 +30,7 @@ def test_method_create_with_all_params_overload_1(self, client: OpenAI) -> None: transcription = client.audio.transcriptions.create( file=b"raw file contents", model="gpt-4o-transcribe", + chunking_strategy="auto", include=["logprobs"], language="language", prompt="prompt", @@ -81,6 +82,7 @@ def test_method_create_with_all_params_overload_2(self, client: OpenAI) -> None: file=b"raw file contents", model="gpt-4o-transcribe", stream=True, + chunking_strategy="auto", include=["logprobs"], language="language", prompt="prompt", @@ -134,6 +136,7 @@ async def test_method_create_with_all_params_overload_1(self, async_client: Asyn transcription = await async_client.audio.transcriptions.create( file=b"raw file contents", model="gpt-4o-transcribe", + chunking_strategy="auto", include=["logprobs"], language="language", prompt="prompt", @@ -185,6 +188,7 @@ async def test_method_create_with_all_params_overload_2(self, async_client: Asyn file=b"raw file contents", model="gpt-4o-transcribe", stream=True, + chunking_strategy="auto", include=["logprobs"], language="language", prompt="prompt",