Merge pull request #6852 from PatrickFarley/openai-audio

OpenAI audio

Author: v-shils

articles/ai-foundry/openai/concepts/models.md

@@ -321,6 +321,7 @@ Details about maximum request tokens and training data are available in the foll
 |`gpt-4o-realtime-preview` (2025-06-03) <br> GPT-4o audio | Audio model for real-time audio processing. |Input: 128,000  <br> Output: 4,096 | October 2023 |
 |`gpt-4o-realtime-preview` (2024-12-17) <br> GPT-4o audio | Audio model for real-time audio processing. |Input: 128,000  <br> Output: 4,096 | October 2023 |
 |`gpt-4o-mini-realtime-preview` (2024-12-17) <br> GPT-4o audio | Audio model for real-time audio processing. |Input: 128,000  <br> Output: 4,096 | October 2023 |
+|`gpt-4o-realtime` (2025-08-28) <br> GPT-4o audio | Audio model for real-time audio processing. |Input: 28,672  <br> Output: 4,096 | October 2023 |
 
 To compare the availability of GPT-4o audio models across all regions, refer to the [models table](#global-standard-model-availability).
 

articles/ai-foundry/openai/includes/realtime-deploy-model.md

@@ -7,12 +7,12 @@ ms.topic: include
 ms.date: 1/21/2025
 ---
 
-To deploy the `gpt-4o-mini-realtime-preview` model in the Azure AI Foundry portal:
+To deploy the `gpt-4o-realtime` model in the Azure AI Foundry portal:
 1. Go to the [Azure AI Foundry portal](https://ai.azure.com/?cid=learnDocs) and create or select your project. 
 1. Select **Models + endpoints** from under **My assets** in the left pane.
 1. Select **+ Deploy model** > **Deploy base model** to open the deployment window. 
-1. Search for and select the `gpt-4o-mini-realtime-preview` model and then select **Confirm**.
+1. Search for and select the `gpt-4o-realtime` model and then select **Confirm**.
 1. Review the deployment details and select **Deploy**.
 1. Follow the wizard to finish deploying the model.
 
-Now that you have a deployment of the `gpt-4o-mini-realtime-preview` model, you can interact with it in the Azure AI Foundry portal **Audio** playground or Realtime API.
+Now that you have a deployment of the `gpt-4o-realtime` model, you can interact with it in the Azure AI Foundry portal **Audio** playground or Realtime API.

articles/ai-foundry/openai/includes/realtime-javascript.md

@@ -12,7 +12,7 @@ ms.date: 3/20/2025
 - An Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services" target="_blank">Create one for free</a>
 - <a href="https://nodejs.org/" target="_blank">Node.js LTS or ESM support.</a>
 - An Azure OpenAI resource created in one of the supported regions. For more information about region availability, see the [models and versions documentation](../concepts/models.md#global-standard-model-availability).
-- Then, you need to deploy a `gpt-4o-mini-realtime-preview` model with your Azure OpenAI resource. For more information, see [Create a resource and deploy a model with Azure OpenAI](../how-to/create-resource.md). 
+- Then, you need to deploy a `gpt-4o-realtime` model with your Azure OpenAI resource. For more information, see [Create a resource and deploy a model with Azure OpenAI](../how-to/create-resource.md). 
 
 ### Microsoft Entra ID prerequisites
 
@@ -72,8 +72,8 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
         // You will need to set these environment variables or edit the following values
         const endpoint = process.env.AZURE_OPENAI_ENDPOINT || "AZURE_OPENAI_ENDPOINT";
         // Required Azure OpenAI deployment name and API version
-        const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || "gpt-4o-mini-realtime-preview";
-        const apiVersion = process.env.OPENAI_API_VERSION || "2025-04-01-preview";
+        const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || "gpt-4o-realtime";
+        const apiVersion = process.env.OPENAI_API_VERSION || "2025-08-28";
         // Keyless authentication 
         const credential = new DefaultAzureCredential();
         const scope = "https://cognitiveservices.azure.com/.default";
@@ -90,8 +90,8 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
             realtimeClient.send({
                 type: "session.update",
                 session: {
-                    modalities: ["text", "audio"],
-                    model: "gpt-4o-mini-realtime-preview",
+                    output_modalities: ["text", "audio"],
+                    model: "gpt-4o-realtime",
                 },
             });
             realtimeClient.send({
@@ -113,12 +113,12 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
             console.log("session created!", event.session);
             console.log();
         });
-        realtimeClient.on("response.text.delta", (event) => process.stdout.write(event.delta));
-        realtimeClient.on("response.audio.delta", (event) => {
+        realtimeClient.on("response.output_text.delta", (event) => process.stdout.write(event.delta));
+        realtimeClient.on("response.output_audio.delta", (event) => {
             const buffer = Buffer.from(event.delta, "base64");
             console.log(`Received ${buffer.length} bytes of audio data.`);
         });
-        realtimeClient.on("response.audio_transcript.delta", (event) => {
+        realtimeClient.on("response.output_audio_transcript.delta", (event) => {
             console.log(`Received text delta:${event.delta}.`);
         });
         realtimeClient.on("response.text.done", () => console.log());
@@ -155,8 +155,8 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
         const endpoint = process.env.AZURE_OPENAI_ENDPOINT || "AZURE_OPENAI_ENDPOINT";
         const apiKey = process.env.AZURE_OPENAI_API_KEY || "Your API key";
         // Required Azure OpenAI deployment name and API version
-        const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || "gpt-4o-mini-realtime-preview";
-        const apiVersion = process.env.OPENAI_API_VERSION || "2025-04-01-preview";
+        const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || "gpt-4o-realtime";
+        const apiVersion = process.env.OPENAI_API_VERSION || "2025-28-08";
         const azureOpenAIClient = new AzureOpenAI({
             apiKey: apiKey,
             apiVersion: apiVersion,
@@ -170,7 +170,7 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
                 type: "session.update",
                 session: {
                     modalities: ["text", "audio"],
-                    model: "gpt-4o-mini-realtime-preview",
+                    model: "gpt-4o-realtime",
                 },
             });
             realtimeClient.send({
@@ -192,12 +192,12 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
             console.log("session created!", event.session);
             console.log();
         });
-        realtimeClient.on("response.text.delta", (event) => process.stdout.write(event.delta));
-        realtimeClient.on("response.audio.delta", (event) => {
+        realtimeClient.on("response.output_text.delta", (event) => process.stdout.write(event.delta));
+        realtimeClient.on("response.output_audio.delta", (event) => {
             const buffer = Buffer.from(event.delta, "base64");
             console.log(`Received ${buffer.length} bytes of audio data.`);
         });
-        realtimeClient.on("response.audio_transcript.delta", (event) => {
+        realtimeClient.on("response.output_audio_transcript.delta", (event) => {
             console.log(`Received text delta:${event.delta}.`);
         });
         realtimeClient.on("response.text.done", () => console.log());

articles/ai-foundry/openai/includes/realtime-portal.md

@@ -13,18 +13,18 @@ ms.date: 3/20/2025
 
 ## Use the GPT-4o real-time audio
 
-To chat with your deployed `gpt-4o-mini-realtime-preview` model in the [Azure AI Foundry](https://ai.azure.com/?cid=learnDocs) **Real-time audio** playground, follow these steps:
+To chat with your deployed `gpt-4o-realtime` model in the [Azure AI Foundry](https://ai.azure.com/?cid=learnDocs) **Real-time audio** playground, follow these steps:
 
-1. Go to the [Azure AI Foundry portal](https://ai.azure.com/?cid=learnDocs) and select your project that has your deployed `gpt-4o-mini-realtime-preview` model.
+1. Go to the [Azure AI Foundry portal](https://ai.azure.com/?cid=learnDocs) and select your project that has your deployed `gpt-4o-realtime` model.
 1. Select **Playgrounds** from the left pane.
 1. Select **Audio playground** > **Try the Audio playground**. 
 
     > [!NOTE]
-    > The **Chat playground** doesn't support the `gpt-4o-mini-realtime-preview` model. Use the **Audio playground** as described in this section.
+    > The **Chat playground** doesn't support the `gpt-4o-realtime` model. Use the **Audio playground** as described in this section.
 
-1. Select your deployed `gpt-4o-mini-realtime-preview` model from the **Deployment** dropdown.
+1. Select your deployed `gpt-4o-realtime` model from the **Deployment** dropdown.
 
-    :::image type="content" source="../media/how-to/real-time/real-time-playground.png" alt-text="Screenshot of the audio playground with the deployed model selected." lightbox="../media/how-to/real-time/real-time-playground.png":::
+    <!--:::image type="content" source="../media/how-to/real-time/real-time-playground.png" alt-text="Screenshot of the audio playground with the deployed model selected." lightbox="../media/how-to/real-time/real-time-playground.png":::-->
 
 1. Optionally, you can edit contents in the **Give the model instructions and context** text box. Give the model instructions about how it should behave and any context it should reference when generating a response. You can describe the assistant's personality, tell it what it should and shouldn't answer, and tell it how to format responses.
 1. Optionally, change settings such as threshold, prefix padding, and silence duration.

articles/ai-foundry/openai/includes/realtime-python.md

@@ -12,7 +12,7 @@ ms.date: 3/20/2025
 - An Azure subscription. <a href="https://azure.microsoft.com/free/ai-services" target="_blank">Create one for free</a>.
 - <a href="https://www.python.org/" target="_blank">Python 3.8 or later version</a>. We recommend using Python 3.10 or later, but having at least Python 3.8 is required. If you don't have a suitable version of Python installed, you can follow the instructions in the [VS Code Python Tutorial](https://code.visualstudio.com/docs/python/python-tutorial#_install-a-python-interpreter) for the easiest way of installing Python on your operating system.
 - An Azure OpenAI resource created in one of the supported regions. For more information about region availability, see the [models and versions documentation](../concepts/models.md#global-standard-model-availability).
-- Then, you need to deploy a `gpt-4o-mini-realtime-preview` model with your Azure OpenAI resource. For more information, see [Create a resource and deploy a model with Azure OpenAI](../how-to/create-resource.md).
+- Then, you need to deploy a `gpt-4o-realtime` model with your Azure OpenAI resource. For more information, see [Create a resource and deploy a model with Azure OpenAI](../how-to/create-resource.md).
 
 ## Microsoft Entra ID prerequisites
 
@@ -109,12 +109,12 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
         client = AsyncAzureOpenAI(
             azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
             azure_ad_token_provider=token_provider,
-            api_version="2025-04-01-preview",
+            api_version="2025-08-28",
         )
         async with client.beta.realtime.connect(
-            model="gpt-4o-realtime-preview",  # name of your deployment
+            model="gpt-4o-realtime",  # name of your deployment
         ) as connection:
-            await connection.session.update(session={"modalities": ["text", "audio"]})  
+            await connection.session.update(session={"output_modalities": ["text", "audio"]})  
             while True:
                 user_input = input("Enter a message: ")
                 if user_input == "q":
@@ -129,15 +129,15 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
                 )
                 await connection.response.create()
                 async for event in connection:
-                    if event.type == "response.text.delta":
+                    if event.type == "response.output_text.delta":
                         print(event.delta, flush=True, end="")
-                    elif event.type == "response.audio.delta":
+                    elif event.type == "response.output_audio.delta":
                         
                         audio_data = base64.b64decode(event.delta)
                         print(f"Received {len(audio_data)} bytes of audio data.")
-                    elif event.type == "response.audio_transcript.delta":
+                    elif event.type == "response.output_audio_transcript.delta":
                         print(f"Received text delta: {event.delta}")
-                    elif event.type == "response.text.done":
+                    elif event.type == "response.output_text.done":
                         print()
                     elif event.type == "response.done":
                         break
@@ -181,12 +181,12 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
         client = AsyncAzureOpenAI(
             azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
             api_key=os.environ["AZURE_OPENAI_API_KEY"],
-            api_version="2025-04-01-preview",
+            api_version="2025-08-28",
         )
         async with client.beta.realtime.connect(
-            model="gpt-4o-realtime-preview",  # deployment name of your model
+            model="gpt-4o-realtime",  # deployment name of your model
         ) as connection:
-            await connection.session.update(session={"modalities": ["text", "audio"]})  
+            await connection.session.update(session={"output_modalities": ["text", "audio"]})  
             while True:
                 user_input = input("Enter a message: ")
                 if user_input == "q":
@@ -201,15 +201,15 @@ For the recommended keyless authentication with Microsoft Entra ID, you need to:
                 )
                 await connection.response.create()
                 async for event in connection:
-                    if event.type == "response.text.delta":
+                    if event.type == "response.output_text.delta":
                         print(event.delta, flush=True, end="")
-                    elif event.type == "response.audio.delta":
+                    elif event.type == "response.output_audio.delta":
                         
                         audio_data = base64.b64decode(event.delta)
                         print(f"Received {len(audio_data)} bytes of audio data.")
-                    elif event.type == "response.audio_transcript.delta":
+                    elif event.type == "response.output_audio_transcript.delta":
                         print(f"Received text delta: {event.delta}")
-                    elif event.type == "response.text.done":
+                    elif event.type == "response.output_text.done":
                         print()
                     elif event.type == "response.done":
                         break