com.rest.elevenlabs 3.5.1 (#121)

StephenHodgson · web-flow · commit c6c049f9bf7e · 2025-03-08T23:48:10.000Z
- Fixed generated clip playback for non-streaming clips
- Updated usages of Task.Delay with WebGL friendly Awaiters.DelayAsync
- Updated TextToSpeechDemo
- Updated Deps
- Updated Unit Tests
diff --git a/Runtime/Common/GeneratedClip.cs b/Runtime/Common/GeneratedClip.cs
@@ -57,62 +57,61 @@ internal GeneratedClip(string id, string text, ReadOnlyMemory<byte> clipData, in
         public Guid TextHash { get; private set; }
 
         [SerializeField]
-        private AudioClip audioClip;
+        private string cachedPath;
 
         [Preserve]
-        public AudioClip AudioClip
+        public string CachedPath => cachedPath;
+
+        public ReadOnlyMemory<byte> ClipData { get; }
+
+        public float[] ClipSamples
         {
             get
             {
-                if (audioClip == null && !ClipData.IsEmpty)
+                if (clipSamples != null)
                 {
-                    var samples = ClipSamples;
-
-                    if (samples is { Length: > 0 })
-                    {
-                        audioClip = AudioClip.Create(Id, samples.Length, 1, SampleRate, false);
-                        audioClip.SetData(samples, 0);
-                    }
+                    return clipSamples;
                 }
 
-                if (audioClip == null)
+                if (ClipData.IsEmpty)
                 {
-                    Debug.LogError($"{nameof(audioClip)} is null, try loading it with LoadCachedAudioClipAsync");
+                    return Array.Empty<float>();
                 }
 
-                return audioClip;
+                clipSamples ??= PCMEncoder.Decode(ClipData.ToArray(), inputSampleRate: SampleRate, outputSampleRate: AudioSettings.outputSampleRate);
+                return clipSamples;
+
             }
         }
+        private float[] clipSamples;
+
+        public int SampleRate { get; }
 
         [SerializeField]
-        private string cachedPath;
+        private AudioClip audioClip;
 
         [Preserve]
-        public string CachedPath => cachedPath;
-
-        public ReadOnlyMemory<byte> ClipData { get; }
-
-        private float[] clipSamples;
-
-        public float[] ClipSamples
+        public AudioClip AudioClip
         {
             get
             {
-                if (!ClipData.IsEmpty)
+                if (audioClip == null &&
+                    ClipSamples is { Length: > 0 })
                 {
-                    clipSamples ??= PCMEncoder.Decode(ClipData.ToArray(), PCMFormatSize.SixteenBit, SampleRate, AudioSettings.outputSampleRate);
+                    audioClip = AudioClip.Create(Id, ClipSamples.Length, 1, AudioSettings.outputSampleRate, false);
+                    audioClip.SetData(ClipSamples, 0);
                 }
-                else if (audioClip != null)
+
+                if (audioClip == null)
                 {
-                    clipSamples = new float[audioClip.samples];
-                    audioClip.GetData(clipSamples, 0);
+                    Debug.LogError($"{nameof(audioClip)} is null, try loading it with LoadCachedAudioClipAsync");
                 }
 
-                return clipSamples;
+                return audioClip;
             }
         }
 
-        public int SampleRate { get; }
+        public float Length => ClipSamples.Length / (float)AudioSettings.outputSampleRate;
 
         public void OnBeforeSerialize() => textHash = TextHash.ToString();
 
@@ -130,6 +129,12 @@ var path when path.EndsWith(".mp3") => AudioType.MPEG,
                 _ => AudioType.UNKNOWN
             };
 
+            if (audioType == AudioType.UNKNOWN)
+            {
+                Debug.LogWarning($"Unable to load cached audio clip at {cachedPath}");
+                return null;
+            }
+
             return await Rest.DownloadAudioClipAsync($"file://{cachedPath}", audioType, cancellationToken: cancellationToken);
         }
     }
diff --git a/Runtime/Dubbing/DubbingEndpoint.cs b/Runtime/Dubbing/DubbingEndpoint.cs
@@ -10,6 +10,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using UnityEngine;
+using Utilities.Async;
 using Utilities.WebRequestRest;
 using Debug = UnityEngine.Debug;
 
@@ -148,7 +149,7 @@ private async Task<DubbingProjectMetadata> WaitForDubbingCompletionAsync(Dubbing
                         Debug.Log($"Dubbing for {dubbingResponse.DubbingId} in progress... Will check status again in {pollingInterval.TotalSeconds} seconds.");
                     }
 
-                    await Task.Delay(pollingInterval, cancellationToken).ConfigureAwait(false);
+                    await Awaiters.DelayAsync(pollingInterval, cancellationToken).ConfigureAwait(true);
                 }
                 else
                 {
diff --git a/Runtime/TextToSpeech/TextToSpeechEndpoint.cs b/Runtime/TextToSpeech/TextToSpeechEndpoint.cs
@@ -180,7 +180,7 @@ public async Task<VoiceClip> TextToSpeechAsync(TextToSpeechRequest request, Func
             var audioData = request.WithTimestamps ? accumulatedPCMData!.ToArray() : response.Data;
             var cachedPath = await SaveAudioToCache(audioData, clipId, request.Voice, request.OutputFormat, request.CacheFormat, cancellationToken).ConfigureAwait(true);
 
-            return new VoiceClip(clipId, request.Text, request.Voice, new ReadOnlyMemory<byte>(audioData), request.OutputFormat.GetSampleRate(), cachedPath)
+            return new VoiceClip(clipId, request.Text, request.Voice, new ReadOnlyMemory<byte>(audioData), frequency, cachedPath)
             {
                 TimestampedTranscriptCharacters = accumulatedTranscriptData?.ToArray() ?? Array.Empty<TimestampedTranscriptCharacter>()
             };
diff --git a/Samples~/TextToSpeech/TextToSpeechDemo.cs b/Samples~/TextToSpeech/TextToSpeechDemo.cs
@@ -4,12 +4,14 @@
 using ElevenLabs.TextToSpeech;
 using ElevenLabs.Voices;
 using System;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using UnityEngine;
 using Utilities.Async;
 using Utilities.Audio;
+using Debug = UnityEngine.Debug;
 
 namespace ElevenLabs.Demo
 {
@@ -63,18 +65,29 @@ private async void Start()
                 }
 
                 var request = new TextToSpeechRequest(voice, message, model: Model.FlashV2_5, outputFormat: OutputFormat.PCM_24000);
+                var stopwatch = Stopwatch.StartNew();
                 var voiceClip = await api.TextToSpeechEndpoint.TextToSpeechAsync(request, async partialClip =>
                 {
                     await streamAudioSource.BufferCallbackAsync(partialClip.ClipSamples);
                 }, cancellationToken: destroyCancellationToken);
-                await new WaitUntil(() => streamAudioSource.IsEmpty || destroyCancellationToken.IsCancellationRequested);
-                destroyCancellationToken.ThrowIfCancellationRequested();
-                ((AudioSource)streamAudioSource).clip = voiceClip.AudioClip;
+                var elapsedTime = (float)stopwatch.Elapsed.TotalSeconds;
+                var playbackTime = voiceClip.Length - elapsedTime;
+
+                if (debug)
+                {
+                    Debug.Log($"Elapsed time: {elapsedTime:F} seconds");
+                    Debug.Log($"voice clip length: {voiceClip.Length:F} seconds");
+                    Debug.Log($"playback time: {playbackTime:F} seconds");
+                }
+
+                await Awaiters.DelayAsync(TimeSpan.FromSeconds(playbackTime + 1f), destroyCancellationToken);
 
                 if (debug)
                 {
                     Debug.Log($"Full clip: {voiceClip.Id}");
                 }
+
+                ((AudioSource)streamAudioSource).PlayOneShot(voiceClip);
             }
             catch (Exception e)
             {
diff --git a/Tests/Test_Fixture_04_TextToSpeechEndpoint.cs b/Tests/Test_Fixture_04_TextToSpeechEndpoint.cs
@@ -21,6 +21,7 @@ public async Task Test_01_TextToSpeech()
             var voiceClip = await ElevenLabsClient.TextToSpeechEndpoint.TextToSpeechAsync(request);
             Assert.NotNull(voiceClip);
             Assert.NotNull(voiceClip.AudioClip);
+            Assert.AreEqual(voiceClip.AudioClip.length, voiceClip.Length, 0.01);
             Debug.Log(voiceClip.Id);
         }
 
@@ -36,9 +37,10 @@ public async Task Test_02_StreamTextToSpeech()
             Assert.NotNull(partialClips);
             Assert.IsNotEmpty(partialClips);
             Assert.NotNull(voiceClip);
-            Assert.IsNotNull(voiceClip.AudioClip);
             Debug.Log(voiceClip.Id);
             Debug.Log(voiceClip.CachedPath);
+            Assert.IsNotNull(voiceClip.AudioClip);
+            Assert.AreEqual(voiceClip.AudioClip.length, voiceClip.Length, 0.01);
         }
 
         [Test]
@@ -50,9 +52,10 @@ public async Task Test_03_TextToSpeech_Transcription()
             var request = new TextToSpeechRequest(voice, "The quick brown fox jumps over the lazy dog.", withTimestamps: true);
             var voiceClip = await ElevenLabsClient.TextToSpeechEndpoint.TextToSpeechAsync(request);
             Assert.NotNull(voiceClip);
-            Assert.NotNull(voiceClip.AudioClip);
             Debug.Log(voiceClip.Id);
             Debug.Log(voiceClip.CachedPath);
+            Assert.NotNull(voiceClip.AudioClip);
+            Assert.AreEqual(voiceClip.AudioClip.length, voiceClip.Length, 0.01);
             Assert.NotNull(voiceClip.TimestampedTranscriptCharacters);
             Assert.IsNotEmpty(voiceClip.TimestampedTranscriptCharacters);
             Debug.Log("| Character | Start Time | End Time |");
@@ -88,9 +91,10 @@ public async Task Test_04_StreamTextToSpeech_Transcription()
             Assert.NotNull(partialClips);
             Assert.IsNotEmpty(partialClips);
             Assert.NotNull(voiceClip);
-            Assert.IsNotNull(voiceClip.AudioClip);
             Debug.Log(voiceClip.Id);
             Debug.Log(voiceClip.CachedPath);
+            Assert.IsNotNull(voiceClip.AudioClip);
+            Assert.AreEqual(voiceClip.AudioClip.length, voiceClip.Length, 0.01);
             Assert.AreEqual(characters.ToArray(), voiceClip.TimestampedTranscriptCharacters);
         }
 
@@ -111,10 +115,11 @@ public async Task Test_05_LanguageEnforced_TextToSpeech()
                 languageCode: "cs");
             var voiceClip = await ElevenLabsClient.TextToSpeechEndpoint.TextToSpeechAsync(request);
             Assert.NotNull(voiceClip);
-            Assert.NotNull(voiceClip.AudioClip);
-            Assert.IsTrue(string.IsNullOrWhiteSpace(voiceClip.CachedPath));
             Debug.Log(voiceClip.Id);
             Debug.Log(voiceClip.CachedPath);
+            Assert.NotNull(voiceClip.AudioClip);
+            Assert.AreEqual(voiceClip.AudioClip.length, voiceClip.Length, 0.01);
+            Assert.IsTrue(string.IsNullOrWhiteSpace(voiceClip.CachedPath));
         }
     }
 }
diff --git a/package.json b/package.json
@@ -3,7 +3,7 @@
   "displayName": "ElevenLabs",
   "description": "A non-official Eleven Labs voice synthesis RESTful client.",
   "keywords": [],
-  "version": "3.5.0",
+  "version": "3.5.1",
   "unity": "2021.3",
   "documentationUrl": "https://github.com/RageAgainstThePixel/com.rest.elevenlabs#documentation",
   "changelogUrl": "https://github.com/RageAgainstThePixel/com.rest.elevenlabs/releases",
@@ -17,10 +17,9 @@
     "url": "https://github.com/StephenHodgson"
   },
   "dependencies": {
-    "com.utilities.rest": "3.3.1",
-    "com.utilities.audio": "2.2.1",
-    "com.utilities.encoder.ogg": "4.2.0",
-    "com.utilities.encoder.wav": "2.2.0"
+    "com.utilities.rest": "3.3.2",
+    "com.utilities.encoder.ogg": "4.2.1",
+    "com.utilities.encoder.wav": "2.2.1"
   },
   "samples": [
     {

Original file line number	Diff line number	Diff line change
`@@ -57,62 +57,61 @@ internal GeneratedClip(string id, string text, ReadOnlyMemory<byte> clipData, in`
`57`	`57`	`public Guid TextHash { get; private set; }`
`58`	`58`
`59`	`59`	`[SerializeField]`
`60`		`- private AudioClip audioClip;`
	`60`	`+ private string cachedPath;`
`61`	`61`
`62`	`62`	`[Preserve]`
`63`		`- public AudioClip AudioClip`
	`63`	`+ public string CachedPath => cachedPath;`
	`64`	`+`
	`65`	`+ public ReadOnlyMemory<byte> ClipData { get; }`
	`66`	`+`
	`67`	`+ public float[] ClipSamples`
`64`	`68`	`{`
`65`	`69`	`get`
`66`	`70`	`{`
`67`		`- if (audioClip == null && !ClipData.IsEmpty)`
	`71`	`+ if (clipSamples != null)`
`68`	`72`	`{`
`69`		`- var samples = ClipSamples;`
`70`		`-`
`71`		`- if (samples is { Length: > 0 })`
`72`		`- {`
`73`		`- audioClip = AudioClip.Create(Id, samples.Length, 1, SampleRate, false);`
`74`		`- audioClip.SetData(samples, 0);`
`75`		`- }`
	`73`	`+ return clipSamples;`
`76`	`74`	`}`
`77`	`75`
`78`		`- if (audioClip == null)`
	`76`	`+ if (ClipData.IsEmpty)`
`79`	`77`	`{`
`80`		`- Debug.LogError($"{nameof(audioClip)} is null, try loading it with LoadCachedAudioClipAsync");`
	`78`	`+ return Array.Empty<float>();`
`81`	`79`	`}`
`82`	`80`
`83`		`- return audioClip;`
	`81`	`+ clipSamples ??= PCMEncoder.Decode(ClipData.ToArray(), inputSampleRate: SampleRate, outputSampleRate: AudioSettings.outputSampleRate);`
	`82`	`+ return clipSamples;`
	`83`	`+`
`84`	`84`	`}`
`85`	`85`	`}`
	`86`	`+ private float[] clipSamples;`
	`87`	`+`
	`88`	`+ public int SampleRate { get; }`
`86`	`89`
`87`	`90`	`[SerializeField]`
`88`		`- private string cachedPath;`
	`91`	`+ private AudioClip audioClip;`
`89`	`92`
`90`	`93`	`[Preserve]`
`91`		`- public string CachedPath => cachedPath;`
`92`		`-`
`93`		`- public ReadOnlyMemory<byte> ClipData { get; }`
`94`		`-`
`95`		`- private float[] clipSamples;`
`96`		`-`
`97`		`- public float[] ClipSamples`
	`94`	`+ public AudioClip AudioClip`
`98`	`95`	`{`
`99`	`96`	`get`
`100`	`97`	`{`
`101`		`- if (!ClipData.IsEmpty)`
	`98`	`+ if (audioClip == null &&`
	`99`	`+ ClipSamples is { Length: > 0 })`
`102`	`100`	`{`
`103`		`- clipSamples ??= PCMEncoder.Decode(ClipData.ToArray(), PCMFormatSize.SixteenBit, SampleRate, AudioSettings.outputSampleRate);`
	`101`	`+ audioClip = AudioClip.Create(Id, ClipSamples.Length, 1, AudioSettings.outputSampleRate, false);`
	`102`	`+ audioClip.SetData(ClipSamples, 0);`
`104`	`103`	`}`
`105`		`- else if (audioClip != null)`
	`104`	`+`
	`105`	`+ if (audioClip == null)`
`106`	`106`	`{`
`107`		`- clipSamples = new float[audioClip.samples];`
`108`		`- audioClip.GetData(clipSamples, 0);`
	`107`	`+ Debug.LogError($"{nameof(audioClip)} is null, try loading it with LoadCachedAudioClipAsync");`
`109`	`108`	`}`
`110`	`109`
`111`		`- return clipSamples;`
	`110`	`+ return audioClip;`
`112`	`111`	`}`
`113`	`112`	`}`
`114`	`113`
`115`		`- public int SampleRate { get; }`
	`114`	`+ public float Length => ClipSamples.Length / (float)AudioSettings.outputSampleRate;`
`116`	`115`
`117`	`116`	`public void OnBeforeSerialize() => textHash = TextHash.ToString();`
`118`	`117`
`@@ -130,6 +129,12 @@ var path when path.EndsWith(".mp3") => AudioType.MPEG,`
`130`	`129`	`_ => AudioType.UNKNOWN`
`131`	`130`	`};`
`132`	`131`
	`132`	`+ if (audioType == AudioType.UNKNOWN)`
	`133`	`+ {`
	`134`	`+ Debug.LogWarning($"Unable to load cached audio clip at {cachedPath}");`
	`135`	`+ return null;`
	`136`	`+ }`
	`137`	`+`
`133`	`138`	`return await Rest.DownloadAudioClipAsync($"file://{cachedPath}", audioType, cancellationToken: cancellationToken);`
`134`	`139`	`}`
`135`	`140`	`}`
Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,7 @@`
`10`	`10`	`using System.Threading;`
`11`	`11`	`using System.Threading.Tasks;`
`12`	`12`	`using UnityEngine;`
	`13`	`+using Utilities.Async;`
`13`	`14`	`using Utilities.WebRequestRest;`
`14`	`15`	`using Debug = UnityEngine.Debug;`
`15`	`16`
`@@ -148,7 +149,7 @@ private async Task<DubbingProjectMetadata> WaitForDubbingCompletionAsync(Dubbing`
`148`	`149`	`Debug.Log($"Dubbing for {dubbingResponse.DubbingId} in progress... Will check status again in {pollingInterval.TotalSeconds} seconds.");`
`149`	`150`	`}`
`150`	`151`
`151`		`- await Task.Delay(pollingInterval, cancellationToken).ConfigureAwait(false);`
	`152`	`+ await Awaiters.DelayAsync(pollingInterval, cancellationToken).ConfigureAwait(true);`
`152`	`153`	`}`
`153`	`154`	`else`
`154`	`155`	`{`
Original file line number	Diff line number	Diff line change
`@@ -180,7 +180,7 @@ public async Task<VoiceClip> TextToSpeechAsync(TextToSpeechRequest request, Func`
`180`	`180`	`var audioData = request.WithTimestamps ? accumulatedPCMData!.ToArray() : response.Data;`
`181`	`181`	`var cachedPath = await SaveAudioToCache(audioData, clipId, request.Voice, request.OutputFormat, request.CacheFormat, cancellationToken).ConfigureAwait(true);`
`182`	`182`
`183`		`- return new VoiceClip(clipId, request.Text, request.Voice, new ReadOnlyMemory<byte>(audioData), request.OutputFormat.GetSampleRate(), cachedPath)`
	`183`	`+ return new VoiceClip(clipId, request.Text, request.Voice, new ReadOnlyMemory<byte>(audioData), frequency, cachedPath)`
`184`	`184`	`{`
`185`	`185`	`TimestampedTranscriptCharacters = accumulatedTranscriptData?.ToArray() ?? Array.Empty<TimestampedTranscriptCharacter>()`
`186`	`186`	`};`