Uso de GPT Realtime API para voz y audio

Obtenga más información sobre la autenticación sin claves y la configuración de variables de entorno.

Obtenga más información sobre cómo buscar claves de API y establecer variables de entorno.

1. Cree el index.js archivo con el código siguiente:

   import OpenAI from 'openai';
   import { OpenAIRealtimeWS } from 'openai/realtime/ws';
   import { DefaultAzureCredential, getBearerTokenProvider } from '@azure/identity';
   import { OpenAIRealtimeError } from 'openai/realtime/internal-base';
   
   let isCreated = false;
   let isConfigured = false;
   let responseDone = false;
   
   // Set this to false, if you want to continue receiving events after an error is received.
   const throwOnError = true;
   
   async function main() {
       // The endpoint of your Azure OpenAI resource is required. You can set it in the AZURE_OPENAI_ENDPOINT
       // environment variable or replace the default value below.
       // You can find it in the Microsoft Foundry portal in the Overview page of your Azure OpenAI resource.
       // Example: https://{your-resource}.openai.azure.com
       const endpoint = process.env.AZURE_OPENAI_ENDPOINT || 'AZURE_OPENAI_ENDPOINT';
       const baseUrl = endpoint.replace(/\/$/, "") + '/openai/v1';
   
       // The deployment name of your Azure OpenAI model is required. You can set it in the AZURE_OPENAI_DEPLOYMENT_NAME
       // environment variable or replace the default value below.
       // You can find it in the Foundry portal in the "Models + endpoints" page of your Azure OpenAI resource.
       // Example: gpt-realtime
       const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || 'gpt-realtime';
   
       // Keyless authentication
       const credential = new DefaultAzureCredential();
       const scope = 'https://ai.azure.com/.default';
       const azureADTokenProvider = getBearerTokenProvider(credential, scope);
       const token = await azureADTokenProvider();
   
       // The APIs are compatible with the OpenAI client library.
       // You can use the OpenAI client library to access the Azure OpenAI APIs.
       // Make sure to set the baseURL and apiKey to use the Azure OpenAI endpoint and token.
       const openAIClient = new OpenAI({
           baseURL: baseUrl,
           apiKey: token,
       });
       const realtimeClient = await OpenAIRealtimeWS.create(openAIClient, {
           model: deploymentName
       });
   
       realtimeClient.on('error', (receivedError) => receiveError(receivedError));
       realtimeClient.on('session.created', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('session.updated', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.output_audio.delta', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.output_audio_transcript.delta', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.done', (receivedEvent) => receiveEvent(receivedEvent));
   
       console.log('Waiting for events...');
       while (!isCreated) {
           console.log('Waiting for session.created event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       // After the session is created, configure it to enable audio input and output.
       const sessionConfig = {
           'type': 'realtime',
           'instructions': 'You are a helpful assistant. You respond by voice and text.',
           'output_modalities': ['audio'],
           'audio': {
               'input': {
                   'transcription': {
                       'model': 'whisper-1'
                   },
                   'format': {
                       'type': 'audio/pcm',
                       'rate': 24000,
                   },
                   'turn_detection': {
                       'type': 'server_vad',
                       'threshold': 0.5,
                       'prefix_padding_ms': 300,
                       'silence_duration_ms': 200,
                       'create_response': true
                   }
               },
               'output': {
                   'voice': 'alloy',
                   'format': {
                       'type': 'audio/pcm',
                       'rate': 24000,
                   }
               }
           }
       };
   
       realtimeClient.send({
           'type': 'session.update',
           'session': sessionConfig
       });
       while (!isConfigured) {
           console.log('Waiting for session.updated event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       // After the session is configured, data can be sent to the session.    
       realtimeClient.send({
           'type': 'conversation.item.create',
           'item': {
               'type': 'message',
               'role': 'user',
               'content': [{
                   type: 'input_text',
                   text: 'Please assist the user.'
               }
               ]
           }
       });
   
       realtimeClient.send({
           type: 'response.create'
       });
   
   
   
       // While waiting for the session to finish, the events can be handled in the event handlers.
       // In this example, we just wait for the first response.done event.
       while (!responseDone) {
           console.log('Waiting for response.done event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       console.log('The sample completed successfully.');
       realtimeClient.close();
   }
   
   function receiveError(err) {
       if (err instanceof OpenAIRealtimeError) {
           console.error('Received an error event.');
           console.error(`Message: ${err.cause.message}`);
           console.error(`Stack: ${err.cause.stack}`);
       }
   
       if (throwOnError) {
           throw err;
       }
   }
   
   function receiveEvent(event) {
       console.log(`Received an event: ${event.type}`);
   
       switch (event.type) {
           case 'session.created':
               console.log(`Session ID: ${event.session.id}`);
               isCreated = true;
               break;
           case 'session.updated':
               console.log(`Session ID: ${event.session.id}`);
               isConfigured = true;
               break;
           case 'response.output_audio_transcript.delta':
               console.log(`Transcript delta: ${event.delta}`);
               break;
           case 'response.output_audio.delta':
               let audioBuffer = Buffer.from(event.delta, 'base64');
               console.log(`Audio delta length: ${audioBuffer.length} bytes`);
               break;
           case 'response.done':
               console.log(`Response ID: ${event.response.id}`);
               console.log(`The final response is: ${event.response.output[0].content[0].transcript}`);
               responseDone = true;
               break;
           default:
               console.warn(`Unhandled event type: ${event.type}`);
       }
   }
   
   main().catch((err) => {
       console.error('The sample encountered an error:', err);
   });
   export {
       main
   };
   

2. Inicie sesión en Azure con el comando siguiente:

   az login
   

3. Ejecute el archivo JavaScript.

   node index.js
   

1. Cree el index.js archivo con el código siguiente:

   import OpenAI from 'openai';
   import { OpenAIRealtimeWS } from 'openai/realtime/ws';
   import { OpenAIRealtimeError } from 'openai/realtime/internal-base';
   
   let isCreated = false;
   let isConfigured = false;
   let responseDone = false;
   
   // Set this to false, if you want to continue receiving events after an error is received.
   const throwOnError = true;
   
   async function main() {
       // The endpoint of your Azure OpenAI resource is required. You can set it in the AZURE_OPENAI_ENDPOINT
       // environment variable or replace the default value below.
       // You can find it in the Foundry portal in the Overview page of your Azure OpenAI resource.
       // Example: https://{your-resource}.openai.azure.com
       const endpoint = process.env.AZURE_OPENAI_ENDPOINT || 'AZURE_OPENAI_ENDPOINT';
       const baseUrl = endpoint.replace(/\/$/, "") + '/openai/v1';
   
       // The deployment name of your Azure OpenAI model is required. You can set it in the AZURE_OPENAI_DEPLOYMENT_NAME
       // environment variable or replace the default value below.
       // You can find it in the Foundry portal in the "Models + endpoints" page of your Azure OpenAI resource.
       // Example: gpt-realtime
       const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || 'gpt-realtime';
   
       // API Key of your Azure OpenAI resource is required. You can set it in the AZURE_OPENAI_API_KEY
       // environment variable or replace the default value below.
       // You can find it in the Foundry portal in the Overview page of your Azure OpenAI resource.
       const token = process.env.AZURE_OPENAI_API_KEY || '<Your API Key>';
   
       // The APIs are compatible with the OpenAI client library.
       // You can use the OpenAI client library to access the Azure OpenAI APIs.
       // Make sure to set the baseURL and apiKey to use the Azure OpenAI endpoint and token.
       const openAIClient = new OpenAI({
           baseURL: baseUrl,
           apiKey: token,
       });
   
       // Due to the current SDK limitation we need to explicitly
       // pass API key as Header
       const realtimeClient = await OpenAIRealtimeWS.create(
           openAIClient, {
           model: deploymentName,
           options: {
               headers: {
                   "api-key": token
               }
           }
       });
   
       realtimeClient.on('error', (receivedError) => receiveError(receivedError));
       realtimeClient.on('session.created', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('session.updated', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.output_audio.delta', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.output_audio_transcript.delta', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.done', (receivedEvent) => receiveEvent(receivedEvent));
   
       console.log('Waiting for events...');
       while (!isCreated) {
           console.log('Waiting for session.created event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       // After the session is created, configure it to enable audio input and output.
       const sessionConfig = {
           'type': 'realtime',
           'instructions': 'You are a helpful assistant. You respond by voice and text.',
           'output_modalities': ['audio'],
           'audio': {
               'input': {
                   'transcription': {
                       'model': 'whisper-1'
                   },
                   'format': {
                       'type': 'audio/pcm',
                       'rate': 24000,
                   },
                   'turn_detection': {
                       'type': 'server_vad',
                       'threshold': 0.5,
                       'prefix_padding_ms': 300,
                       'silence_duration_ms': 200,
                       'create_response': true
                   }
               },
               'output': {
                   'voice': 'alloy',
                   'format': {
                       'type': 'audio/pcm',
                       'rate': 24000,
                   }
               }
           }
       };
   
       realtimeClient.send({
           'type': 'session.update',
           'session': sessionConfig
       });
       while (!isConfigured) {
           console.log('Waiting for session.updated event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       // After the session is configured, data can be sent to the session.
       realtimeClient.send({
           'type': 'conversation.item.create',
           'item': {
               'type': 'message',
               'role': 'user',
               'content': [{
                   type: 'input_text',
                   text: 'Please assist the user.'
               }
               ]
           }
       });
   
       realtimeClient.send({
           type: 'response.create'
       });
   
       // While waiting for the session to finish, the events can be handled in the event handlers.
       // In this example, we just wait for the first response.done event.
       while (!responseDone) {
           console.log('Waiting for response.done event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       console.log('The sample completed successfully.');
       realtimeClient.close();
   }
   
   function receiveError(err) {
       if (err instanceof OpenAIRealtimeError) {
           console.error('Received an error event.');
           console.error(`Message: ${err.cause.message}`);
           console.error(`Stack: ${err.cause.stack}`);
       }
   
       if (throwOnError) {
           throw err;
       }
   }
   
   function receiveEvent(event) {
       console.log(`Received an event: ${event.type}`);
   
       switch (event.type) {
           case 'session.created':
               console.log(`Session ID: ${event.session.id}`);
               isCreated = true;
               break;
           case 'session.updated':
               console.log(`Session ID: ${event.session.id}`);
               isConfigured = true;
               break;
           case 'response.output_audio_transcript.delta':
               console.log(`Transcript delta: ${event.delta}`);
               break;
           case 'response.output_audio.delta':
               let audioBuffer = Buffer.from(event.delta, 'base64');
               console.log(`Audio delta length: ${audioBuffer.length} bytes`);
               break;
           case 'response.done':
               console.log(`Response ID: ${event.response.id}`);
               console.log(`The final response is: ${event.response.output[0].content[0].transcript}`);
               responseDone = true;
               break;
           default:
               console.warn(`Unhandled event type: ${event.type}`);
       }
   }
   
   main().catch((err) => {
       console.error('The sample encountered an error:', err);
   });
   export {
       main
   };
   

2. Ejecute el archivo JavaScript.

   node index.js
   

1. Cree el audio-in-audio-out.js archivo con el código siguiente:

   import OpenAI from 'openai';
   import { OpenAIRealtimeWS } from 'openai/realtime/ws';
   import { DefaultAzureCredential, getBearerTokenProvider } from '@azure/identity';
   import fs from 'fs';
   
   async function main() {
       const endpoint = process.env.AZURE_OPENAI_ENDPOINT || 'AZURE_OPENAI_ENDPOINT';
       const baseUrl = endpoint.replace(/\/$/, "") + '/openai/v1';
       const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || 'gpt-realtime';
   
       // Keyless authentication
       const credential = new DefaultAzureCredential();
       const scope = 'https://ai.azure.com/.default';
       const azureADTokenProvider = getBearerTokenProvider(credential, scope);
       const token = await azureADTokenProvider();
   
       const openAIClient = new OpenAI({
           baseURL: baseUrl,
           apiKey: token,
       });
   
       const inputAudioFile = 'input.pcm';
       const outputAudioFile = 'output.pcm';
       let outputAudio = Buffer.alloc(0);
       let isConfigured = false;
       let responseDone = false;
   
       const realtimeClient = await OpenAIRealtimeWS.create(openAIClient, {
           model: deploymentName
       });
   
       realtimeClient.on('session.updated', () => {
           console.log('Session configured for audio input/output.');
           isConfigured = true;
       });
   
       realtimeClient.on('response.audio.delta', (event) => {
           const audioChunk = Buffer.from(event.delta, 'base64');
           outputAudio = Buffer.concat([outputAudio, audioChunk]);
       });
   
       realtimeClient.on('response.audio_transcript.delta', (event) => {
           process.stdout.write(event.delta);
       });
   
       realtimeClient.on('conversation.item.input_audio_transcription.completed', (event) => {
           console.log(`\n[User said]: ${event.transcript}`);
       });
   
       realtimeClient.on('response.done', () => {
           responseDone = true;
       });
   
       realtimeClient.on('error', (err) => {
           console.error('Error:', err);
       });
   
       // Wait for session to be created
       await new Promise(resolve => setTimeout(resolve, 500));
   
       // Configure session for audio
       realtimeClient.send({
           type: 'session.update',
           session: {
               instructions: 'You are a helpful assistant. Respond conversationally.',
               input_audio_format: 'pcm16',
               output_audio_format: 'pcm16',
               input_audio_transcription: { model: 'whisper-1' },
               turn_detection: {
                   type: 'server_vad',
                   threshold: 0.5,
                   prefix_padding_ms: 300,
                   silence_duration_ms: 500,
                   create_response: true,
               },
               voice: 'alloy',
           }
       });
   
       while (!isConfigured) {
           await new Promise(resolve => setTimeout(resolve, 100));
       }
   
       // Read and send audio file in chunks
       console.log(`Reading audio from ${inputAudioFile}...`);
       const audioData = fs.readFileSync(inputAudioFile);
   
       const chunkSize = 4800; // 100ms of audio at 24kHz, 16-bit
       for (let i = 0; i < audioData.length; i += chunkSize) {
           const chunk = audioData.slice(i, i + chunkSize);
           realtimeClient.send({
               type: 'input_audio_buffer.append',
               audio: chunk.toString('base64')
           });
           await new Promise(resolve => setTimeout(resolve, 50));
       }
   
       console.log('Audio sent. Waiting for response...');
   
       // Commit the audio buffer
       realtimeClient.send({ type: 'input_audio_buffer.commit' });
   
       // Wait for response to complete
       while (!responseDone) {
           await new Promise(resolve => setTimeout(resolve, 100));
       }
   
       // Save output audio
       if (outputAudio.length > 0) {
           fs.writeFileSync(outputAudioFile, outputAudio);
           console.log(`\n\nSaved ${outputAudio.length} bytes of audio to ${outputAudioFile}`);
           console.log('Play with: ffplay -f s16le -ar 24000 -ac 1 output.pcm');
       }
   
       realtimeClient.close();
   }
   
   main().catch(console.error);
   

2. Inicie sesión en Azure:

   az login
   

3. Ejecute el archivo JavaScript:

   node audio-in-audio-out.js
   

1. Cree el audio-in-audio-out.js archivo con el código siguiente:

   import OpenAI from 'openai';
   import { OpenAIRealtimeWS } from 'openai/realtime/ws';
   import fs from 'fs';
   
   async function main() {
       const endpoint = process.env.AZURE_OPENAI_ENDPOINT || 'AZURE_OPENAI_ENDPOINT';
       const baseUrl = endpoint.replace(/\/$/, "") + '/openai/v1';
       const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || 'gpt-realtime';
       const token = process.env.AZURE_OPENAI_API_KEY || '<Your API Key>';
   
       const openAIClient = new OpenAI({
           baseURL: baseUrl,
           apiKey: token,
       });
   
       const inputAudioFile = 'input.pcm';
       const outputAudioFile = 'output.pcm';
       let outputAudio = Buffer.alloc(0);
       let isConfigured = false;
       let responseDone = false;
   
       const realtimeClient = await OpenAIRealtimeWS.create(openAIClient, {
           model: deploymentName,
           options: {
               headers: { "api-key": token }
           }
       });
   
       realtimeClient.on('session.updated', () => {
           console.log('Session configured for audio input/output.');
           isConfigured = true;
       });
   
       realtimeClient.on('response.audio.delta', (event) => {
           const audioChunk = Buffer.from(event.delta, 'base64');
           outputAudio = Buffer.concat([outputAudio, audioChunk]);
       });
   
       realtimeClient.on('response.audio_transcript.delta', (event) => {
           process.stdout.write(event.delta);
       });
   
       realtimeClient.on('conversation.item.input_audio_transcription.completed', (event) => {
           console.log(`\n[User said]: ${event.transcript}`);
       });
   
       realtimeClient.on('response.done', () => {
           responseDone = true;
       });
   
       realtimeClient.on('error', (err) => {
           console.error('Error:', err);
       });
   
       // Wait for session to be created
       await new Promise(resolve => setTimeout(resolve, 500));
   
       // Configure session for audio
       realtimeClient.send({
           type: 'session.update',
           session: {
               instructions: 'You are a helpful assistant. Respond conversationally.',
               input_audio_format: 'pcm16',
               output_audio_format: 'pcm16',
               input_audio_transcription: { model: 'whisper-1' },
               turn_detection: {
                   type: 'server_vad',
                   threshold: 0.5,
                   prefix_padding_ms: 300,
                   silence_duration_ms: 500,
                   create_response: true,
               },
               voice: 'alloy',
           }
       });
   
       while (!isConfigured) {
           await new Promise(resolve => setTimeout(resolve, 100));
       }
   
       // Read and send audio file in chunks
       console.log(`Reading audio from ${inputAudioFile}...`);
       const audioData = fs.readFileSync(inputAudioFile);
   
       const chunkSize = 4800; // 100ms of audio at 24kHz, 16-bit
       for (let i = 0; i < audioData.length; i += chunkSize) {
           const chunk = audioData.slice(i, i + chunkSize);
           realtimeClient.send({
               type: 'input_audio_buffer.append',
               audio: chunk.toString('base64')
           });
           await new Promise(resolve => setTimeout(resolve, 50));
       }
   
       console.log('Audio sent. Waiting for response...');
   
       // Commit the audio buffer
       realtimeClient.send({ type: 'input_audio_buffer.commit' });
   
       // Wait for response to complete
       while (!responseDone) {
           await new Promise(resolve => setTimeout(resolve, 100));
       }
   
       // Save output audio
       if (outputAudio.length > 0) {
           fs.writeFileSync(outputAudioFile, outputAudio);
           console.log(`\n\nSaved ${outputAudio.length} bytes of audio to ${outputAudioFile}`);
           console.log('Play with: ffplay -f s16le -ar 24000 -ac 1 output.pcm');
       }
   
       realtimeClient.close();
   }
   
   main().catch(console.error);
   

2. Ejecute el archivo JavaScript:

   node audio-in-audio-out.js
   

Obtenga más información sobre la autenticación sin claves y la configuración de variables de entorno.

Obtenga más información sobre cómo buscar claves de API y establecer variables de entorno.

1. Cree el text-in-audio-out.py archivo con el código siguiente:

   import os
   import base64
   import asyncio
   from openai import AsyncOpenAI
   from azure.identity import DefaultAzureCredential, get_bearer_token_provider
   
   async def main() -> None:
       """
       When prompted for user input, type a message and hit enter to send it to the model.
       Enter "q" to quit the conversation.
       """
   
       credential = DefaultAzureCredential()
       token_provider = get_bearer_token_provider(credential, "https://ai.azure.com/.default")
       token = token_provider()
   
       # The endpoint of your Azure OpenAI resource is required. You can set it in the AZURE_OPENAI_ENDPOINT
       # environment variable.
       # You can find it in the Microsoft Foundry portal in the Overview page of your Azure OpenAI resource.
       # Example: https://{your-resource}.openai.azure.com
       endpoint = os.environ["AZURE_OPENAI_ENDPOINT"]
   
       # The deployment name of the model you want to use is required. You can set it in the AZURE_OPENAI_DEPLOYMENT_NAME
       # environment variable.
       # You can find it in the Foundry portal in the "Models + endpoints" page of your Azure OpenAI resource.
       # Example: gpt-realtime
       deployment_name = os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"]
   
       base_url = endpoint.replace("https://", "wss://").rstrip("/") + "/openai/v1"
   
       # The APIs are compatible with the OpenAI client library.
       # You can use the OpenAI client library to access the Azure OpenAI APIs.
       # Make sure to set the baseURL and apiKey to use the Azure OpenAI endpoint and token.
       client = AsyncOpenAI(
           websocket_base_url=base_url,
           api_key=token
       )
       async with client.realtime.connect(
           model=deployment_name,
       ) as connection:
           # after the connection is created, configure the session.
           await connection.session.update(session={
               "type": "realtime",
               "instructions": "You are a helpful assistant. You respond by voice and text.",
               "output_modalities": ["audio"],
               "audio": {
                   "input": {
                       "transcription": {
                           "model": "whisper-1",
                       },
                       "format": {
                           "type": "audio/pcm",
                           "rate": 24000,
                       },
                       "turn_detection": {
                           "type": "server_vad",
                           "threshold": 0.5,
                           "prefix_padding_ms": 300,
                           "silence_duration_ms": 200,
                           "create_response": True,
                       }
                   },
                   "output": {
                       "voice": "alloy",
                       "format": {
                           "type": "audio/pcm",
                           "rate": 24000,
                       }
                   }
               }
           })
   
           # After the session is configured, data can be sent to the session.
           while True:
               user_input = input("Enter a message: ")
               if user_input == "q":
                   print("Stopping the conversation.")
                   break
   
               await connection.conversation.item.create(
                   item={
                       "type": "message",
                       "role": "user",
                       "content": [{"type": "input_text", "text": user_input}],
                   }
               )
               await connection.response.create()
               async for event in connection:
                   if event.type == "response.output_text.delta":
                       print(event.delta, flush=True, end="")
                   elif event.type == "session.created":
                       print(f"Session ID: {event.session.id}")
                   elif event.type == "response.output_audio.delta":
                       audio_data = base64.b64decode(event.delta)
                       print(f"Received {len(audio_data)} bytes of audio data.")
                   elif event.type == "response.output_audio_transcript.delta":
                       print(f"Received text delta: {event.delta}")
                   elif event.type == "response.output_text.done":
                       print()
                   elif event.type == "error":
                       print("Received an error event.")
                       print(f"Error code: {event.error.code}")
                       print(f"Error Event ID: {event.error.event_id}")
                       print(f"Error message: {event.error.message}")
                   elif event.type == "response.done":
                       break
   
       print("Conversation ended.")
       credential.close()
   
   asyncio.run(main())
   

2. Inicie sesión en Azure con el comando siguiente:

   az login
   

3. Ejecute el archivo Python.

   python text-in-audio-out.py
   

4. Cuando se le solicite la entrada del usuario, escriba un mensaje y presione entrar para enviarlo al modelo. Escriba "q" para salir de la conversación.

1. Cree el text-in-audio-out.py archivo con el código siguiente:

   import os
   import base64
   import asyncio
   from openai import AsyncOpenAI
   
   async def main() -> None:
       """
       When prompted for user input, type a message and hit enter to send it to the model.
       Enter "q" to quit the conversation.
       """
   
       # The endpoint of your Azure OpenAI resource is required. You can set it in the AZURE_OPENAI_ENDPOINT
       # environment variable.
       # You can find it in the Foundry portal in the Overview page of your Azure OpenAI resource.
       # Example: https://{your-resource}.openai.azure.com
       endpoint = os.environ["AZURE_OPENAI_ENDPOINT"]
       base_url = endpoint.replace("https://", "wss://").rstrip("/") + "/openai/v1"
   
       # The deployment name of the model you want to use is required. You can set it in the AZURE_OPENAI_DEPLOYMENT_NAME
       # environment variable.
       # You can find it in the Foundry portal in the "Models + endpoints" page of your Azure OpenAI resource.
       # Example: gpt-realtime
       deployment_name = os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"]
   
       # API Key of your Azure OpenAI resource is required. You can set it in the AZURE_OPENAI_API_KEY
       # environment variable.
       # You can find it in the Foundry portal in the Overview page of your Azure OpenAI resource.
       token=os.environ["AZURE_OPENAI_API_KEY"]
   
       # The APIs are compatible with the OpenAI client library.
       # You can use the OpenAI client library to access the Azure OpenAI APIs.
       # Make sure to set the baseURL and apiKey to use the Azure OpenAI endpoint and token.
       client = AsyncOpenAI(
           websocket_base_url=base_url,
           api_key=token
       )
       async with client.realtime.connect(
           model=deployment_name,
       ) as connection:
           # after the connection is created, configure the session.
           await connection.session.update(session={
               "type": "realtime",
               "instructions": "You are a helpful assistant. You respond by voice and text.",
               "output_modalities": ["audio"],
               "audio": {
                   "input": {
                       "transcription": {
                           "model": "whisper-1",
                       },
                       "format": {
                           "type": "audio/pcm",
                           "rate": 24000,
                       },
                       "turn_detection": {
                           "type": "server_vad",
                           "threshold": 0.5,
                           "prefix_padding_ms": 300,
                           "silence_duration_ms": 200,
                           "create_response": True,
                       }
                   },
                   "output": {
                       "voice": "alloy",
                       "format": {
                           "type": "audio/pcm",
                           "rate": 24000,
                       }
                   }
               }
           })
   
           # After the session is configured, data can be sent to the session.
           while True:
               user_input = input("Enter a message: ")
               if user_input == "q":
                   print("Stopping the conversation.")
                   break
   
               await connection.conversation.item.create(
                   item={
                       "type": "message",
                       "role": "user",
                       "content": [{"type": "input_text", "text": user_input}],
                   }
               )
               await connection.response.create()
               async for event in connection:
                   if event.type == "response.output_text.delta":
                       print(event.delta, flush=True, end="")
                   elif event.type == "session.created":
                       print(f"Session ID: {event.session.id}")
                   elif event.type == "response.output_audio.delta":
                       audio_data = base64.b64decode(event.delta)
                       print(f"Received {len(audio_data)} bytes of audio data.")
                   elif event.type == "response.output_audio_transcript.delta":
                       print(f"Received text delta: {event.delta}")
                   elif event.type == "response.output_text.done":
                       print()
                   elif event.type == "error":
                       print("Received an error event.")
                       print(f"Error code: {event.error.code}")
                       print(f"Error Event ID: {event.error.event_id}")
                       print(f"Error message: {event.error.message}")
                   elif event.type == "response.done":
                       break
   
       print("Conversation ended.")
   
   asyncio.run(main())
   

2. Ejecute el archivo Python.

   python text-in-audio-out.py
   

3. Cuando se le solicite la entrada del usuario, escriba un mensaje y presione entrar para enviarlo al modelo. Escriba "q" para salir de la conversación.

1. Cree el audio-in-audio-out.py archivo con el código siguiente:

   import os
   import base64
   import asyncio
   from openai import AsyncOpenAI
   from azure.identity import DefaultAzureCredential, get_bearer_token_provider
   
   async def main() -> None:
       """
       Send audio from a file to the Realtime API and receive an audio response.
       The input file should be in PCM16 format at 24kHz mono.
       """
   
       credential = DefaultAzureCredential()
       token_provider = get_bearer_token_provider(credential, "https://ai.azure.com/.default")
       token = token_provider()
   
       endpoint = os.environ["AZURE_OPENAI_ENDPOINT"]
       deployment_name = os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"]
       base_url = endpoint.replace("https://", "wss://").rstrip("/") + "/openai/v1"
   
       # Path to your input audio file (PCM16, 24kHz, mono)
       input_audio_file = "input.pcm"
       output_audio_file = "output.pcm"
   
       client = AsyncOpenAI(
           websocket_base_url=base_url,
           api_key=token
       )
   
       async with client.realtime.connect(model=deployment_name) as connection:
           # Configure the session for audio input and output
           await connection.session.update(session={
               "instructions": "You are a helpful assistant. Respond conversationally.",
               "input_audio_format": "pcm16",
               "output_audio_format": "pcm16",
               "input_audio_transcription": {"model": "whisper-1"},
               "turn_detection": {
                   "type": "server_vad",
                   "threshold": 0.5,
                   "prefix_padding_ms": 300,
                   "silence_duration_ms": 500,
                   "create_response": True,
               },
               "voice": "alloy",
           })
   
           # Read and send audio file in chunks
           print(f"Reading audio from {input_audio_file}...")
           with open(input_audio_file, "rb") as f:
               audio_data = f.read()
   
           # Send audio in chunks (the Realtime API expects base64-encoded audio)
           chunk_size = 4800  # 100ms of audio at 24kHz, 16-bit
           for i in range(0, len(audio_data), chunk_size):
               chunk = audio_data[i:i + chunk_size]
               await connection.input_audio_buffer.append(audio=base64.b64encode(chunk).decode())
               # Small delay to simulate real-time streaming
               await asyncio.sleep(0.05)
   
           print("Audio sent. Waiting for response...")
   
           # Commit the audio buffer to signal end of input
           await connection.input_audio_buffer.commit()
   
           # Collect audio response
           output_audio = bytearray()
           transcript = ""
   
           async for event in connection:
               if event.type == "response.audio.delta":
                   audio_chunk = base64.b64decode(event.delta)
                   output_audio.extend(audio_chunk)
               elif event.type == "response.audio_transcript.delta":
                   transcript += event.delta
                   print(event.delta, end="", flush=True)
               elif event.type == "conversation.item.input_audio_transcription.completed":
                   print(f"\n[User said]: {event.transcript}")
               elif event.type == "response.done":
                   break
   
           # Save output audio
           if output_audio:
               with open(output_audio_file, "wb") as f:
                   f.write(output_audio)
               print(f"\n\nSaved {len(output_audio)} bytes of audio to {output_audio_file}")
               print("Play with: ffplay -f s16le -ar 24000 -ac 1 output.pcm")
   
       credential.close()
   
   asyncio.run(main())
   

2. Inicie sesión en Azure:

   az login
   

3. Ejecute el archivo Python:

   python audio-in-audio-out.py
   

1. Cree el audio-in-audio-out.py archivo con el código siguiente:

   import os
   import base64
   import asyncio
   from openai import AsyncOpenAI
   
   async def main() -> None:
       """
       Send audio from a file to the Realtime API and receive an audio response.
       The input file should be in PCM16 format at 24kHz mono.
       """
   
       endpoint = os.environ["AZURE_OPENAI_ENDPOINT"]
       deployment_name = os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"]
       token = os.environ["AZURE_OPENAI_API_KEY"]
       base_url = endpoint.replace("https://", "wss://").rstrip("/") + "/openai/v1"
   
       # Path to your input audio file (PCM16, 24kHz, mono)
       input_audio_file = "input.pcm"
       output_audio_file = "output.pcm"
   
       client = AsyncOpenAI(
           websocket_base_url=base_url,
           api_key=token
       )
   
       async with client.realtime.connect(model=deployment_name) as connection:
           # Configure the session for audio input and output
           await connection.session.update(session={
               "instructions": "You are a helpful assistant. Respond conversationally.",
               "input_audio_format": "pcm16",
               "output_audio_format": "pcm16",
               "input_audio_transcription": {"model": "whisper-1"},
               "turn_detection": {
                   "type": "server_vad",
                   "threshold": 0.5,
                   "prefix_padding_ms": 300,
                   "silence_duration_ms": 500,
                   "create_response": True,
               },
               "voice": "alloy",
           })
   
           # Read and send audio file in chunks
           print(f"Reading audio from {input_audio_file}...")
           with open(input_audio_file, "rb") as f:
               audio_data = f.read()
   
           # Send audio in chunks (the Realtime API expects base64-encoded audio)
           chunk_size = 4800  # 100ms of audio at 24kHz, 16-bit
           for i in range(0, len(audio_data), chunk_size):
               chunk = audio_data[i:i + chunk_size]
               await connection.input_audio_buffer.append(audio=base64.b64encode(chunk).decode())
               # Small delay to simulate real-time streaming
               await asyncio.sleep(0.05)
   
           print("Audio sent. Waiting for response...")
   
           # Commit the audio buffer to signal end of input
           await connection.input_audio_buffer.commit()
   
           # Collect audio response
           output_audio = bytearray()
           transcript = ""
   
           async for event in connection:
               if event.type == "response.audio.delta":
                   audio_chunk = base64.b64decode(event.delta)
                   output_audio.extend(audio_chunk)
               elif event.type == "response.audio_transcript.delta":
                   transcript += event.delta
                   print(event.delta, end="", flush=True)
               elif event.type == "conversation.item.input_audio_transcription.completed":
                   print(f"\n[User said]: {event.transcript}")
               elif event.type == "response.done":
                   break
   
           # Save output audio
           if output_audio:
               with open(output_audio_file, "wb") as f:
                   f.write(output_audio)
               print(f"\n\nSaved {len(output_audio)} bytes of audio to {output_audio_file}")
               print("Play with: ffplay -f s16le -ar 24000 -ac 1 output.pcm")
   
   asyncio.run(main())
   

2. Ejecute el archivo Python:

   python audio-in-audio-out.py
   

Obtenga más información sobre la autenticación sin claves y la configuración de variables de entorno.

Obtenga más información sobre cómo buscar claves de API y establecer variables de entorno.

1. Cree el index.ts archivo con el código siguiente:

   import OpenAI from 'openai';
   import { OpenAIRealtimeWS } from 'openai/realtime/ws';
   import { OpenAIRealtimeError } from 'openai/realtime/internal-base';
   import { DefaultAzureCredential, getBearerTokenProvider } from "@azure/identity";
   import { RealtimeSessionCreateRequest } from 'openai/resources/realtime/realtime';
   
   let isCreated = false;
   let isConfigured = false;
   let responseDone = false;
   
   // Set this to false, if you want to continue receiving events after an error is received.
   const throwOnError = true;
   
   async function main(): Promise<void> {
       // The endpoint of your Azure OpenAI resource is required. You can set it in the AZURE_OPENAI_ENDPOINT
       // environment variable or replace the default value below.
       // You can find it in the Microsoft Foundry portal in the Overview page of your Azure OpenAI resource.
       // Example: https://{your-resource}.openai.azure.com
       const endpoint = process.env.AZURE_OPENAI_ENDPOINT || 'AZURE_OPENAI_ENDPOINT';
       const baseUrl = endpoint.replace(/\/$/, "") + '/openai/v1';
   
       // The deployment name of your Azure OpenAI model is required. You can set it in the AZURE_OPENAI_DEPLOYMENT_NAME
       // environment variable or replace the default value below.
       // You can find it in the Foundry portal in the "Models + endpoints" page of your Azure OpenAI resource.
       // Example: gpt-realtime
       const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || 'gpt-realtime';
   
       // Keyless authentication
       const credential = new DefaultAzureCredential();
       const scope = "https://ai.azure.com/.default";
       const azureADTokenProvider = getBearerTokenProvider(credential, scope);
       const token = await azureADTokenProvider();
   
       // The APIs are compatible with the OpenAI client library.
       // You can use the OpenAI client library to access the Azure OpenAI APIs.
       // Make sure to set the baseURL and apiKey to use the Azure OpenAI endpoint and token.
       const openAIClient = new OpenAI({
           baseURL: baseUrl,
           apiKey: token,
       });
       const realtimeClient = await OpenAIRealtimeWS.create(openAIClient, { model: deploymentName });
   
       realtimeClient.on('error', (receivedError) => receiveError(receivedError));
       realtimeClient.on('session.created', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('session.updated', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.output_audio.delta', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.output_audio_transcript.delta', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.done', (receivedEvent) => receiveEvent(receivedEvent));
   
       console.log('Waiting for events...');
       while (!isCreated) {
           console.log('Waiting for session.created event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       // After the session is created, configure it to enable audio input and output.
       const sessionConfig: RealtimeSessionCreateRequest = {
           'type': 'realtime',
           'instructions': 'You are a helpful assistant. You respond by voice and text.',
           'output_modalities': ['audio'],
           'audio': {
               'input': {
                   'transcription': {
                       'model': 'whisper-1'
                   },
                   'format': {
                       'type': 'audio/pcm',
                       'rate': 24000,
                   },
                   'turn_detection': {
                       'type': 'server_vad',
                       'threshold': 0.5,
                       'prefix_padding_ms': 300,
                       'silence_duration_ms': 200,
                       'create_response': true
                   }
               },
               'output': {
                   'voice': 'alloy',
                   'format': {
                       'type': 'audio/pcm',
                       'rate': 24000,
                   }
               }
           }
       };
   
       realtimeClient.send({ 'type': 'session.update', 'session': sessionConfig });
   
       while (!isConfigured) {
           console.log('Waiting for session.updated event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       // After the session is configured, data can be sent to the session.
       realtimeClient.send({
           'type': 'conversation.item.create',
           'item': {
               'type': 'message',
               'role': 'user',
               'content': [{ type: 'input_text', text: 'Please assist the user.' }]
           }
       });
   
       realtimeClient.send({ type: 'response.create' });
   
       // While waiting for the session to finish, the events can be handled in the event handlers.
       // In this example, we just wait for the first response.done event. 
       while (!responseDone) {
           console.log('Waiting for response.done event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       console.log('The sample completed successfully.');
       realtimeClient.close();
   }
   
   function receiveError(errorEvent: OpenAIRealtimeError): void {
       if (errorEvent instanceof OpenAIRealtimeError) {
           console.error('Received an error event.');
           console.error(`Message: ${errorEvent.message}`);
           console.error(`Stack: ${errorEvent.stack}`); errorEvent
       }
   
       if (throwOnError) {
           throw errorEvent;
       }
   }
   
   function receiveEvent(event: any): void {
       console.log(`Received an event: ${event.type}`);
   
       switch (event.type) {
           case 'session.created':
               console.log(`Session ID: ${event.session.id}`);
               isCreated = true;
               break;
           case 'session.updated':
               console.log(`Session ID: ${event.session.id}`);
               isConfigured = true;
               break;
           case 'response.output_audio_transcript.delta':
               console.log(`Transcript delta: ${event.delta}`);
               break;
           case 'response.output_audio.delta':
               let audioBuffer = Buffer.from(event.delta, 'base64');
               console.log(`Audio delta length: ${audioBuffer.length} bytes`);
               break;
           case 'response.done':
               console.log(`Response ID: ${event.response.id}`);
               console.log(`The final response is: ${event.response.output[0].content[0].transcript}`);
               responseDone = true;
               break;
           default:
               console.warn(`Unhandled event type: ${event.type}`);
       }
   }
   
   main().catch((err) => {
       console.error("The sample encountered an error:", err);
   });
   
   export { main };    
   

2. Cree el tsconfig.json archivo para transpile el código TypeScript y copie el código siguiente para ECMAScript.

   {
       "compilerOptions": {
         "module": "NodeNext",
         "target": "ES2022", // Supports top-level await
         "moduleResolution": "NodeNext",
         "skipLibCheck": true, // Avoid type errors from node_modules
         "strict": true // Enable strict type-checking options
       },
       "include": ["*.ts"]
   }
   

3. Instalar definiciones de tipo para Node

   npm i --save-dev @types/node
   

4. Transpile de TypeScript a JavaScript.

   tsc
   

5. Inicie sesión en Azure con el comando siguiente:

   az login
   

6. Ejecute el código con el siguiente comando:

   node index.js
   

1. Cree el index.ts archivo con el código siguiente:

   import OpenAI from 'openai';
   import { OpenAIRealtimeWS } from 'openai/realtime/ws';
   import { OpenAIRealtimeError } from 'openai/realtime/internal-base';
   import { RealtimeSessionCreateRequest } from 'openai/resources/realtime/realtime';
   
   let isCreated = false;
   let isConfigured = false;
   let responseDone = false;
   
   // Set this to false, if you want to continue receiving events after an error is received.
   const throwOnError = true;
   
   async function main(): Promise<void> {
       // The endpoint of your Azure OpenAI resource is required. You can set it in the AZURE_OPENAI_ENDPOINT
       // environment variable or replace the default value below.
       // You can find it in the Foundry portal in the Overview page of your Azure OpenAI resource.
       // Example: https://{your-resource}.openai.azure.com
       const endpoint = process.env.AZURE_OPENAI_ENDPOINT || 'AZURE_OPENAI_ENDPOINT';
       const baseUrl = endpoint.replace(/\/$/, "") + '/openai/v1';
   
       // The deployment name of your Azure OpenAI model is required. You can set it in the AZURE_OPENAI_DEPLOYMENT_NAME
       // environment variable or replace the default value below.
       // You can find it in the Foundry portal in the "Models + endpoints" page of your Azure OpenAI resource.
       // Example: gpt-realtime
       const deploymentName = process.env.AZURE_OPENAI_DEPLOYMENT_NAME || 'gpt-realtime';
   
       // API Key of your Azure OpenAI resource is required. You can set it in the AZURE_OPENAI_API_KEY
       // environment variable or replace the default value below.
       // You can find it in the Foundry portal in the Overview page of your Azure OpenAI resource.
       const token = process.env.AZURE_OPENAI_API_KEY || '<Your API Key>';
   
       // The APIs are compatible with the OpenAI client library.
       // You can use the OpenAI client library to access the Azure OpenAI APIs.
       // Make sure to set the baseURL and apiKey to use the Azure OpenAI endpoint and token.
       const openAIClient = new OpenAI({
           baseURL: baseUrl,
           apiKey: token,
       });
   
       // Due to the current SDK limitation we need to explicitly
       // pass API key as Header
       const realtimeClient = await OpenAIRealtimeWS.create(
           openAIClient, {
           model: deploymentName,
           options: {
               headers: {
                   "api-key": token
               }
           }
       });
   
       realtimeClient.on('error', (receivedError) => receiveError(receivedError));
       realtimeClient.on('session.created', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('session.updated', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.output_audio.delta', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.output_audio_transcript.delta', (receivedEvent) => receiveEvent(receivedEvent));
       realtimeClient.on('response.done', (receivedEvent) => receiveEvent(receivedEvent));
   
       console.log('Waiting for events...');
       while (!isCreated) {
           console.log('Waiting for session.created event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       // After the session is created, configure it to enable audio input and output.
       const sessionConfig: RealtimeSessionCreateRequest = {
           'type': 'realtime',
           'instructions': 'You are a helpful assistant. You respond by voice and text.',
           'output_modalities': ['audio'],
           'audio': {
               'input': {
                   'transcription': {
                       'model': 'whisper-1'
                   },
                   'format': {
                       'type': 'audio/pcm',
                       'rate': 24000,
                   },
                   'turn_detection': {
                       'type': 'server_vad',
                       'threshold': 0.5,
                       'prefix_padding_ms': 300,
                       'silence_duration_ms': 200,
                       'create_response': true
                   }
               },
               'output': {
                   'voice': 'alloy',
                   'format': {
                       'type': 'audio/pcm',
                       'rate': 24000,
                   }
               }
           }
       };
   
       realtimeClient.send({
           'type': 'session.update',
           'session': sessionConfig
       });
       while (!isConfigured) {
           console.log('Waiting for session.updated event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       // After the session is configured, data can be sent to the session.    
       realtimeClient.send({
           'type': 'conversation.item.create',
           'item': {
               'type': 'message',
               'role': 'user',
               'content': [{
                   type: 'input_text',
                   text: 'Please assist the user.'
               }
               ]
           }
       });
   
       realtimeClient.send({
           type: 'response.create'
       });
   
       // While waiting for the session to finish, the events can be handled in the event handlers.
       // In this example, we just wait for the first response.done event.
       while (!responseDone) {
           console.log('Waiting for response.done event...');
           await new Promise((resolve) => setTimeout(resolve, 100));
       }
   
       console.log('The sample completed successfully.');
       realtimeClient.close();
   }
   
   function receiveError(errorEvent: OpenAIRealtimeError): void {
       if (errorEvent instanceof OpenAIRealtimeError) {
           console.error('Received an error event.');
           console.error(`Message: ${errorEvent.message}`);
           console.error(`Stack: ${errorEvent.stack}`);
           errorEvent
       }
   
       if (throwOnError) {
           throw errorEvent;
       }
   }
   
   function receiveEvent(event: any): void {
       console.log(`Received an event: ${event.type}`);
   
       switch (event.type) {
           case 'session.created':
               console.log(`Session ID: ${event.session.id}`);
               isCreated = true;
               break;
           case 'session.updated':
               console.log(`Session ID: ${event.session.id}`);
               isConfigured = true;
               break;
           case 'response.output_audio_transcript.delta':
               console.log(`Transcript delta: ${event.delta}`);
               break;
           case 'response.output_audio.delta':
               let audioBuffer = Buffer.from(event.delta, 'base64');
               console.log(`Audio delta length: ${audioBuffer.length} bytes`);
               break;
           case 'response.done':
               console.log(`Response ID: ${event.response.id}`);
               console.log(`The final response is: ${event.response.output[0].content[0].transcript}`);
               responseDone = true;
               break;
           default:
               console.warn(`Unhandled event type: ${event.type}`);
       }
   }
   
   main().catch((err) => {
       console.error("The sample encountered an error:", err);
   });
   
   export {
       main
   };
   

2. Cree el tsconfig.json archivo para transpile el código TypeScript y copie el código siguiente para ECMAScript.

   {
       "compilerOptions": {
         "module": "NodeNext",
         "target": "ES2022", // Supports top-level await
         "moduleResolution": "NodeNext",
         "skipLibCheck": true, // Avoid type errors from node_modules
         "strict": true // Enable strict type-checking options
       },
       "include": ["*.ts"]
   }
   

3. Instalar definiciones de tipo para Node

   npm i --save-dev @types/node
   

4. Transpile de TypeScript a JavaScript.

   tsc
   

5. Ejecute el código con el siguiente comando:

   node index.js