Add dist/ and lib/ folders needed by Realtime translation example (#1769)

Author: erikakettleson-openai

examples/voice_solutions/one_way_translation_using_realtime_api.mdx

@@ -7,8 +7,7 @@ This cookbook demonstrates how to use OpenAI's [ Realtime API](https://platform.
 A real-world use case for this demo is a multilingual, conversational translation where a speaker talks into the speaker app and listeners hear translations in their selected native language via the listener app. Imagine a conference room with a speaker talking in English and a participant with headphones in choosing to listen to a Tagalog translation. Due to the current turn-based nature of audio models, the speaker must pause briefly to allow the model to process and translate speech. However, as models become faster and more efficient, this latency will decrease significantly and the translation will become more seamless.
 
 
-Let's explore the main functionalities and code snippets that illustrate how the app works. You can find the code in the [accompanying repo](https://github.com/openai/openai-cookbook/tree/main/examples/voice_solutions/one_way_translation_using_realtime_api/README.md
-) if you want to run the app locally.
+Let's explore the main functionalities and code snippets that illustrate how the app works. You can find the code in the [accompanying repo](https://github.com/openai/openai-cookbook/tree/main/examples/voice_solutions/one_way_translation_using_realtime_api) if you want to run the app locally.
 
 ## High Level Architecture Overview
 

examples/voice_solutions/one_way_translation_using_realtime_api/relay-server/lib/relay.js

@@ -0,0 +1,84 @@
+import { WebSocketServer } from 'ws';
+import { RealtimeClient } from '@openai/realtime-api-beta';
+
+export class RealtimeRelay {
+  constructor(apiKey) {
+    this.apiKey = apiKey;
+    this.sockets = new WeakMap();
+    this.wss = null;
+  }
+
+  listen(port) {
+    this.wss = new WebSocketServer({ port });
+    this.wss.on('connection', this.connectionHandler.bind(this));
+    this.log(`Listening on ws://localhost:${port}`);
+  }
+
+  async connectionHandler(ws, req) {
+    if (!req.url) {
+      this.log('No URL provided, closing connection.');
+      ws.close();
+      return;
+    }
+
+    const url = new URL(req.url, `http://${req.headers.host}`);
+    const pathname = url.pathname;
+
+    if (pathname !== '/') {
+      this.log(`Invalid pathname: "${pathname}"`);
+      ws.close();
+      return;
+    }
+
+    // Instantiate new client
+    this.log(`Connecting with key "${this.apiKey.slice(0, 3)}..."`);
+    const client = new RealtimeClient({ apiKey: this.apiKey });
+
+    // Relay: OpenAI Realtime API Event -> Browser Event
+    client.realtime.on('server.*', (event) => {
+      this.log(`Relaying "${event.type}" to Client`);
+      ws.send(JSON.stringify(event));
+    });
+    client.realtime.on('close', () => ws.close());
+
+    // Relay: Browser Event -> OpenAI Realtime API Event
+    // We need to queue data waiting for the OpenAI connection
+    const messageQueue = [];
+    const messageHandler = (data) => {
+      try {
+        const event = JSON.parse(data);
+        this.log(`Relaying "${event.type}" to OpenAI`);
+        client.realtime.send(event.type, event);
+      } catch (e) {
+        console.error(e.message);
+        this.log(`Error parsing event from client: ${data}`);
+      }
+    };
+    ws.on('message', (data) => {
+      if (!client.isConnected()) {
+        messageQueue.push(data);
+      } else {
+        messageHandler(data);
+      }
+    });
+    ws.on('close', () => client.disconnect());
+
+    // Connect to OpenAI Realtime API
+    try {
+      this.log(`Connecting to OpenAI...`);
+      await client.connect();
+    } catch (e) {
+      this.log(`Error connecting to OpenAI: ${e.message}`);
+      ws.close();
+      return;
+    }
+    this.log(`Connected to OpenAI successfully!`);
+    while (messageQueue.length) {
+      messageHandler(messageQueue.shift());
+    }
+  }
+
+  log(...args) {
+    console.log(`[RealtimeRelay]`, ...args);
+  }
+}

examples/voice_solutions/one_way_translation_using_realtime_api/src/lib/wavtools/dist/index.d.ts

@@ -0,0 +1,6 @@
+import { AudioAnalysis } from './lib/analysis/audio_analysis.js';
+import { WavPacker } from './lib/wav_packer.js';
+import { WavStreamPlayer } from './lib/wav_stream_player.js';
+import { WavRecorder } from './lib/wav_recorder.js';
+export { AudioAnalysis, WavPacker, WavStreamPlayer, WavRecorder };
+//# sourceMappingURL=index.d.ts.map

examples/voice_solutions/one_way_translation_using_realtime_api/src/lib/wavtools/dist/index.d.ts.map

@@ -0,0 +1,70 @@
+/**
+ * Output of AudioAnalysis for the frequency domain of the audio
+ * @typedef {Object} AudioAnalysisOutputType
+ * @property {Float32Array} values Amplitude of this frequency between {0, 1} inclusive
+ * @property {number[]} frequencies Raw frequency bucket values
+ * @property {string[]} labels Labels for the frequency bucket values
+ */
+/**
+ * Analyzes audio for visual output
+ * @class
+ */
+export class AudioAnalysis {
+    /**
+     * Retrieves frequency domain data from an AnalyserNode adjusted to a decibel range
+     * returns human-readable formatting and labels
+     * @param {AnalyserNode} analyser
+     * @param {number} sampleRate
+     * @param {Float32Array} [fftResult]
+     * @param {"frequency"|"music"|"voice"} [analysisType]
+     * @param {number} [minDecibels] default -100
+     * @param {number} [maxDecibels] default -30
+     * @returns {AudioAnalysisOutputType}
+     */
+    static getFrequencies(analyser: AnalyserNode, sampleRate: number, fftResult?: Float32Array, analysisType?: "frequency" | "music" | "voice", minDecibels?: number, maxDecibels?: number): AudioAnalysisOutputType;
+    /**
+     * Creates a new AudioAnalysis instance for an HTMLAudioElement
+     * @param {HTMLAudioElement} audioElement
+     * @param {AudioBuffer|null} [audioBuffer] If provided, will cache all frequency domain data from the buffer
+     * @returns {AudioAnalysis}
+     */
+    constructor(audioElement: HTMLAudioElement, audioBuffer?: AudioBuffer | null);
+    fftResults: any[];
+    audio: HTMLAudioElement;
+    context: any;
+    analyser: any;
+    sampleRate: any;
+    audioBuffer: any;
+    /**
+     * Gets the current frequency domain data from the playing audio track
+     * @param {"frequency"|"music"|"voice"} [analysisType]
+     * @param {number} [minDecibels] default -100
+     * @param {number} [maxDecibels] default -30
+     * @returns {AudioAnalysisOutputType}
+     */
+    getFrequencies(analysisType?: "frequency" | "music" | "voice", minDecibels?: number, maxDecibels?: number): AudioAnalysisOutputType;
+    /**
+     * Resume the internal AudioContext if it was suspended due to the lack of
+     * user interaction when the AudioAnalysis was instantiated.
+     * @returns {Promise<true>}
+     */
+    resumeIfSuspended(): Promise<true>;
+}
+/**
+ * Output of AudioAnalysis for the frequency domain of the audio
+ */
+export type AudioAnalysisOutputType = {
+    /**
+     * Amplitude of this frequency between {0, 1} inclusive
+     */
+    values: Float32Array;
+    /**
+     * Raw frequency bucket values
+     */
+    frequencies: number[];
+    /**
+     * Labels for the frequency bucket values
+     */
+    labels: string[];
+};
+//# sourceMappingURL=audio_analysis.d.ts.map

examples/voice_solutions/one_way_translation_using_realtime_api/src/lib/wavtools/dist/lib/analysis/audio_analysis.d.ts

@@ -0,0 +1,9 @@
+/**
+ * All note frequencies from 1st to 8th octave
+ * in format "A#8" (A#, 8th octave)
+ */
+export const noteFrequencies: any[];
+export const noteFrequencyLabels: any[];
+export const voiceFrequencies: any[];
+export const voiceFrequencyLabels: any[];
+//# sourceMappingURL=constants.d.ts.map

examples/voice_solutions/one_way_translation_using_realtime_api/src/lib/wavtools/dist/lib/analysis/audio_analysis.d.ts.map

@@ -0,0 +1,58 @@
+/**
+ * Raw wav audio file contents
+ * @typedef {Object} WavPackerAudioType
+ * @property {Blob} blob
+ * @property {string} url
+ * @property {number} channelCount
+ * @property {number} sampleRate
+ * @property {number} duration
+ */
+/**
+ * Utility class for assembling PCM16 "audio/wav" data
+ * @class
+ */
+export class WavPacker {
+    /**
+     * Converts Float32Array of amplitude data to ArrayBuffer in Int16Array format
+     * @param {Float32Array} float32Array
+     * @returns {ArrayBuffer}
+     */
+    static floatTo16BitPCM(float32Array: Float32Array): ArrayBuffer;
+    /**
+     * Concatenates two ArrayBuffers
+     * @param {ArrayBuffer} leftBuffer
+     * @param {ArrayBuffer} rightBuffer
+     * @returns {ArrayBuffer}
+     */
+    static mergeBuffers(leftBuffer: ArrayBuffer, rightBuffer: ArrayBuffer): ArrayBuffer;
+    /**
+     * Packs data into an Int16 format
+     * @private
+     * @param {number} size 0 = 1x Int16, 1 = 2x Int16
+     * @param {number} arg value to pack
+     * @returns
+     */
+    private _packData;
+    /**
+     * Packs audio into "audio/wav" Blob
+     * @param {number} sampleRate
+     * @param {{bitsPerSample: number, channels: Array<Float32Array>, data: Int16Array}} audio
+     * @returns {WavPackerAudioType}
+     */
+    pack(sampleRate: number, audio: {
+        bitsPerSample: number;
+        channels: Array<Float32Array>;
+        data: Int16Array;
+    }): WavPackerAudioType;
+}
+/**
+ * Raw wav audio file contents
+ */
+export type WavPackerAudioType = {
+    blob: Blob;
+    url: string;
+    channelCount: number;
+    sampleRate: number;
+    duration: number;
+};
+//# sourceMappingURL=wav_packer.d.ts.map