NVIDIA Pipecat SDK 0.2.0 release changes

Author: dharmendrach

.gitignore

@@ -25,3 +25,6 @@ output/
 
 # Ignore docs
 docs/build/
+
+# Ignore .DS_Store
+.DS_Store

CHANGELOG.md

@@ -1,7 +1,39 @@
-# NVIDIA Pipecat 0.1.0 (23 April 2025)
+## Changelog
+All notable changes to this project will be documented in this file.
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.2.0] - 2025-06-17
+
+### Added
+- Support for deepseek, mistral-ai, and llama-nemotron models in Nvidia LLM Service
+- Support for BotSpeakingFrame in animation graph service
+
+### Changed
+- Upgraded Riva Client version to 2.20.0
+- Upgraded to pipecat 0.0.68
+- Improved animation graph stream handling
+- Improved task cancellation support in NVIDIA LLM and NVIDIA RAG Service
+
+### Fixed
+- Fixed transcription synchronization for multiple final ASR transcripts
+- Fixed edge case where mouth of avatar would not close
+- Fixed animation stream handling for broken streams
+- Fixed Elevenlabs edge case issues with multi-lingual use cases
+- Fixed chunk truncation issues in RAG Service
+- Fixed dangling tasks and pipeline cleanup issues
+
+## [0.1.1] - 2025-04-30
+
+### Fixed
+
+- `RivaTTSService` doesn't work with `nvidia-riva-client 2.19.1` version due to breaking changes, updated `pyproject.toml` to use `2.19.0` version only.
+
+
+## [0.1.0] - 2025-04-23
 The NVIDIA Pipecat library augments the Pipecat framework by adding additional frame processors and services, as well as new multimodal frames to enhance avatar interactions. This is the first release of the NVIDIA Pipecat library.
 
-## New Features
+### Added
 
 - Added Pipecat services for [Riva ASR (Automatic Speech Recognition)](https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-overview.html#), [Riva TTS (Text to Speech)](https://docs.nvidia.com/deeplearning/riva/user-guide/docs/tts/tts-overview.html), and [Riva NMT (Neural Machine Translation)](https://docs.nvidia.com/deeplearning/riva/user-guide/docs/translation/translation-overview.html) models.
 - Added Pipecat frames, processors, and services to support multimodal avatar interactions and use cases. This includes `Audio2Face3DService`, `AnimationGraphService`, `FacialGestureProviderProcessor`, and `PostureProviderProcessor`.
@@ -13,7 +45,6 @@ The NVIDIA Pipecat library augments the Pipecat framework by adding additional f
 - Released source code for the voice assistant example using `nvidia-pipecat`, along with the `pipecat-ai` library service, to showcase NVIDIA services with `ACETransport`.
 
 
-## Improvements
+### Changed
 
 - Added `ElevenLabsTTSServiceWithEndOfSpeech`, an extended version of the ElevenLabs TTS service with end-of-speech events for usage in avatar interactions.
-

examples/speech-to-speech/bot.py

@@ -11,12 +11,10 @@
 from fastapi.staticfiles import StaticFiles
 from pipecat.audio.vad.silero import SileroVADAnalyzer
 from pipecat.frames.frames import LLMMessagesFrame
-
-# Uncomment the following line if you want to use ElevenLabsTTS
-# from pipecat.services.elevenlabs import ElevenLabsTTSService
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
 
 from nvidia_pipecat.pipeline.ace_pipeline_runner import ACEPipelineRunner, PipelineMetadata
 
@@ -29,12 +27,17 @@
     BotTranscriptSynchronization,
     UserTranscriptSynchronization,
 )
+from nvidia_pipecat.services.elevenlabs import ElevenLabsTTSServiceWithEndOfSpeech
 from nvidia_pipecat.services.nvidia_llm import NvidiaLLMService
-from nvidia_pipecat.services.riva_speech import RivaASRService, RivaTTSService
+from nvidia_pipecat.services.riva_speech import RivaASRService
 from nvidia_pipecat.transports.network.ace_fastapi_websocket import ACETransport, ACETransportParams
 from nvidia_pipecat.transports.services.ace_controller.routers.websocket_router import router as websocket_router
+
+# from nvidia_pipecat.services.riva_speech import RivaTTSService
 from nvidia_pipecat.utils.logging import setup_default_ace_logging
 
+# from nvidia_pipecat.serializers.ace_websocket import ACEWebSocketSerializer
+
 load_dotenv(override=True)
 
 setup_default_ace_logging(level="DEBUG")
@@ -52,16 +55,14 @@ async def create_pipeline_task(pipeline_metadata: PipelineMetadata):
     transport = ACETransport(
         websocket=pipeline_metadata.websocket,
         params=ACETransportParams(
-            vad_enabled=True,
             vad_analyzer=SileroVADAnalyzer(),
-            vad_audio_passthrough=True,
+            # serializer=ACEWebSocketSerializer(),
         ),
     )
 
     llm = NvidiaLLMService(
         api_key=os.getenv("NVIDIA_API_KEY"),
         model="meta/llama-3.1-8b-instruct",
-        base_url=None,
     )
 
     stt = RivaASRService(
@@ -71,27 +72,32 @@ async def create_pipeline_task(pipeline_metadata: PipelineMetadata):
         sample_rate=16000,
         model="parakeet-1.1b-en-US-asr-streaming-silero-vad-asr-bls-ensemble",
     )
-    tts = RivaTTSService(
-        server="localhost:50051",
-        api_key=os.getenv("NVIDIA_API_KEY"),
-        voice_id="English-US.Female-1",
-        language="en-US",
-        quality=20,
+    # Uncomment the following if you want to use Riva TTS (make sure to comment out ElevenLabsTTS below)
+    # tts = RivaTTSService(
+    #     server="localhost:50051",
+    #     api_key=os.getenv("NVIDIA_API_KEY"),
+    #     voice_id="English-US.Female-1",
+    #     model="fastpitch-hifigan-tts",
+    #     language="en-US",
+    #     zero_shot_quality=20,
+    # )
+
+    tts = ElevenLabsTTSServiceWithEndOfSpeech(
+        api_key=os.getenv("ELEVENLABS_API_KEY"),
+        voice_id=os.getenv("ELEVENLABS_VOICE_ID", "cgSgspJ2msm6clMCkdW9"),
         sample_rate=16000,
-        model="fastpitch-hifigan-tts",
+        model="eleven_flash_v2_5",
+        param=ElevenLabsTTSService.InputParams(
+            stability=0.3,
+            speed=0.97,
+            similarity_boost=0.85,
+        ),
     )
+
     # Used to synchronize the user and bot transcripts in the UI
     stt_transcript_synchronization = UserTranscriptSynchronization()
     tts_transcript_synchronization = BotTranscriptSynchronization()
 
-    # Uncomment the following if you want to use ElevenLabsTTS (make sure to comment out Riva TTS below)
-    # tts = ElevenLabsTTSService(
-    #     api_key=os.getenv("ELEVENLABS_API_KEY"),
-    #     voice_id=os.getenv("ELEVENLABS_VOICE_ID", "EXAVITQu4vr4xnSDxMaL"),
-    #     sample_rate=16000,
-    #     model = "eleven_flash_v2_5",
-    # )
-
     messages = [
         {
             "role": "system",
@@ -156,8 +162,8 @@ async def on_client_connected(transport, client):
 
 app = FastAPI()
 app.include_router(websocket_router)
-runner = ACEPipelineRunner(pipeline_callback=create_pipeline_task)
+runner = ACEPipelineRunner.create_instance(pipeline_callback=create_pipeline_task)
 app.mount("/static", StaticFiles(directory=os.path.join(os.path.dirname(__file__), "../static")), name="static")
 
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=8100)
+    uvicorn.run("bot:app", host="0.0.0.0", port=8100, workers=1)

flake.lock

@@ -2,7 +2,7 @@
   description = "NVIDIA ACE Pipecat SDK";
 
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-24.11";
+    nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable";
     flake-utils.url = "github:numtide/flake-utils";
     pyproject-nix = {
       url = "github:pyproject-nix/pyproject.nix";
@@ -49,6 +49,49 @@
           numba = prev.numba.overrideAttrs (old: {
             buildInputs = (old.buildInputs or []) ++ [pkgs.tbb_2021_11];
           });
+          semantic-version = prev.semantic-version.overrideAttrs (old: {
+            nativeBuildInputs =
+              old.nativeBuildInputs
+              ++ final.resolveBuildSystem {
+                setuptools = [];
+                wheel = [];
+              };
+          });
+
+          setuptools-scm = prev.setuptools-scm.overrideAttrs (old: {
+            nativeBuildInputs =
+              old.nativeBuildInputs
+              ++ final.resolveBuildSystem {
+                setuptools = [];
+                wheel = [];
+              };
+          });
+
+          setuptools-rust = prev.setuptools-rust.overrideAttrs (old: {
+            nativeBuildInputs =
+              old.nativeBuildInputs
+              ++ final.resolveBuildSystem {
+                setuptools = [];
+                wheel = [];
+              };
+          });
+
+          libcst = prev.libcst.overrideAttrs (old: {
+            nativeBuildInputs =
+              old.nativeBuildInputs
+              ++ final.resolveBuildSystem {
+                setuptools = [];
+                wheel = [];
+              };
+          });
+          nvidia-pipecat = prev.nvidia-pipecat.overrideAttrs (old: {
+            nativeBuildInputs =
+              old.nativeBuildInputs
+              ++ final.resolveBuildSystem {
+                hatchling = [];
+                editables = [];
+              };
+          });
         };
 
         # Use Python 3.12 from nixpkgs

flake.nix

@@ -1,6 +1,6 @@
 [project]
 name = "nvidia-pipecat"
-version = "0.1.0"
+version = "0.2.0"
 description = "NVIDIA ACE Pipecat SDK"
 readme = "NVIDIA_PIPECAT.md"
 license = { file = "LICENSE" }
@@ -12,10 +12,8 @@ dependencies = [
     "av>=13.0.0",
     "fastapi>=0.115.7",
     "hatchling>=1.27.0",
-    "nvidia-riva-client>=2.18.0",
     "onnxruntime>=1.20.1",
     "openai>=1.58.1",
-    "pipecat-ai==0.0.57",
     "sentence-transformers>=3.3.1",
     "torch>=2.5.1",
     "python-dotenv>=1.0.1",
@@ -30,6 +28,8 @@ dependencies = [
     "nvidia-animation-graph==1.1.0",
     "opentelemetry-sdk>=1.31.0",
     "opentelemetry-instrumentation-grpc>=0.52b0",
+    "nvidia-riva-client==2.20.0",
+    "pipecat-ai==0.0.68",
 ]
 
 [build-system]
@@ -40,7 +40,7 @@ build-backend = "hatchling.build"
 packages = ["src/nvidia_pipecat"]
 
 [tool.uv.sources]
-torch = { index = "pytorch" }
+torch = { index = "pytorch", marker = "sys_platform != 'darwin'" }
 
 [[tool.uv.index]]
 name = "pytorch"