diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 0dd30f9e5b..faca8c03f3 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -33,7 +33,7 @@ jobs: - name: Install dependencies run: | - uv sync --group dev --extra anthropic --extra aws --extra google --extra langchain --extra websocket + uv sync --group dev --extra anthropic --extra aws --extra google --extra langchain --extra livekit --extra websocket - name: Run tests with coverage run: | diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8e58845e40..459725da70 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -37,7 +37,7 @@ jobs: - name: Install dependencies run: | - uv sync --group dev --extra anthropic --extra aws --extra google --extra langchain --extra websocket + uv sync --group dev --extra anthropic --extra aws --extra google --extra langchain --extra livekit --extra websocket - name: Test with pytest run: | diff --git a/.gitignore b/.gitignore index 2e5da188e7..426f9be3a8 100644 --- a/.gitignore +++ b/.gitignore @@ -51,4 +51,7 @@ docs/api/_build/ docs/api/api # uv -.python-version \ No newline at end of file +.python-version + +# Pipecat +whisker_setup.py \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index f874df5e93..1d70f0682f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,129 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +## [0.0.100] - 2026-01-20 + +### Added + +- Added Hathora service to support Hathora-hosted TTS and STT models (only + non-streaming) + (PR [#3169](https://github.com/pipecat-ai/pipecat/pull/3169)) + +- Added `CambTTSService`, using Camb.ai's TTS integration with MARS models + (mars-flash, mars-pro, mars-instruct) for high-quality text-to-speech + synthesis. + (PR [#3349](https://github.com/pipecat-ai/pipecat/pull/3349)) + +- Added the `additional_headers` param to `WebsocketClientParams`, allowing + `WebsocketClientTransport` to send custom headers on connect, for cases such + as authentication. + (PR [#3461](https://github.com/pipecat-ai/pipecat/pull/3461)) + +- Added `UserIdleController` for detecting user idle state, integrated into + `LLMUserAggregator` and `UserTurnProcessor` via optional `user_idle_timeout` + parameter. Emits `on_user_turn_idle` event for application-level handling. + Deprecated `UserIdleProcessor` in favor of the new compositional approach. + (PR [#3482](https://github.com/pipecat-ai/pipecat/pull/3482)) + +- Added `on_user_mute_started` and `on_user_mute_stopped` event handlers to + `LLMUserAggregator` for tracking user mute state changes. + (PR [#3490](https://github.com/pipecat-ai/pipecat/pull/3490)) + +### Changed + +- Enhanced interruption handling in `AsyncAITTSService` by supporting + multi-context WebSocket sessions for more robust context management. + (PR [#3287](https://github.com/pipecat-ai/pipecat/pull/3287)) + +- Throttle `UserSpeakingFrame` to broadcast at most every 200ms instead of on + every audio chunk, reducing frame processing overhead during user speech. + (PR [#3483](https://github.com/pipecat-ai/pipecat/pull/3483)) + +### Deprecated + +- For consistency with other package names, we just deprecated + `pipecat.turns.mute` (introduced in Pipecat 0.0.99) in favor of + `pipecat.turns.user_mute`. + (PR [#3479](https://github.com/pipecat-ai/pipecat/pull/3479)) + +### Fixed + +- Corrected TTFB metric calculation in `AsyncAIHttpTTSService`. + (PR [#3287](https://github.com/pipecat-ai/pipecat/pull/3287)) + +- Fixed an issue where the "bot-llm-text" RTVI event would not fire for + realtime (speech-to-speech) services: + + - `AWSNovaSonicLLMService` + - `GeminiLiveLLMService` + - `OpenAIRealtimeLLMService` + - `GrokRealtimeLLMService` + + The issue was that these services weren't pushing `LLMTextFrame`s. Now + they do. + (PR [#3446](https://github.com/pipecat-ai/pipecat/pull/3446)) + +- Fixed an issue where `on_user_turn_stop_timeout` could fire while a user is + talking when using `ExternalUserTurnStrategies`. + (PR [#3454](https://github.com/pipecat-ai/pipecat/pull/3454)) + +- Fixed an issue where user turn start strategies were not being reset after a + user turn started, causing incorrect strategy behavior. + (PR [#3455](https://github.com/pipecat-ai/pipecat/pull/3455)) + +- Fixed `MinWordsUserTurnStartStrategy` to not aggregate transcriptions, + preventing incorrect turn starts when words are spoken with pauses between + them. + (PR [#3462](https://github.com/pipecat-ai/pipecat/pull/3462)) + +- Fixed an issue where Grok Realtime would error out when running with + SmallWebRTC transport. + (PR [#3480](https://github.com/pipecat-ai/pipecat/pull/3480)) + +- Fixed a `Mem0MemoryService` issue where passing `async_mode: true` was + causing an error. See + https://docs.mem0.ai/platform/features/async-mode-default-change. + (PR [#3484](https://github.com/pipecat-ai/pipecat/pull/3484)) + +- Fixed `AWSNovaSonicLLMService.reset_conversation()`, which would previously + error out. Now it successfully reconnects and "rehydrates" from the context + object. + (PR [#3486](https://github.com/pipecat-ai/pipecat/pull/3486)) + +- Fixed `AzureTTSService` transcript formatting issues: + - Punctuation now appears without extra spaces (e.g., "Hello!" instead of + "Hello !") + - CJK languages (Chinese, Japanese, Korean) no longer have unwanted spaces + between characters + (PR [#3489](https://github.com/pipecat-ai/pipecat/pull/3489)) + +- Fixed an issue where `UninterruptibleFrame` frames would not be preserved in + some cases. + (PR [#3494](https://github.com/pipecat-ai/pipecat/pull/3494)) + +- Fixed memory leak in `LiveKitTransport` when `video_in_enabled` is `False`. + (PR [#3499](https://github.com/pipecat-ai/pipecat/pull/3499)) + +- Fixed an issue in `AIService` where unhandled exceptions in `start()`, + `stop()`, or `cancel()` implementations would prevent `process_frame()` to + continue and therefore `StartFrame`, `EndFrame`, or `CancelFrame` from being + pushed downstream, causing the pipeline to not start or stop properly. + (PR [#3503](https://github.com/pipecat-ai/pipecat/pull/3503)) + +- Moved `NVIDIATTSService` and `NVIDIASTTService` client initialization from + constructor to `start()` for better error handling. + (PR [#3504](https://github.com/pipecat-ai/pipecat/pull/3504)) + +- Optimized `NVIDIATTSService` to process incoming audio frames immediately. + (PR [#3509](https://github.com/pipecat-ai/pipecat/pull/3509)) + +- Optimized `NVIDIASTTService` by removing unnecessary queue and task. + (PR [#3509](https://github.com/pipecat-ai/pipecat/pull/3509)) + +- Fixed a `CambTTSService` issue where client was being initialized in the + constructor which wouldn't allow for proper Pipeline error handling. + (PR [#3511](https://github.com/pipecat-ai/pipecat/pull/3511)) + ## [0.0.99] - 2026-01-13 ### Added @@ -450,6 +573,7 @@ PIPECAT_SETUP_FILES="setup1.py:setup.py:..."`). Each file must define a ### Fixed - Improved error handling in `ElevenLabsRealtimeSTTService` + (PR [#3233](https://github.com/pipecat-ai/pipecat/pull/3233)) - Fixed an issue in `ElevenLabsRealtimeSTTService` causing an infinite loop that blocks the process if the websocket disconnects due to an error @@ -500,6 +624,9 @@ PIPECAT_SETUP_FILES="setup1.py:setup.py:..."`). Each file must define a guard was set. (PR [#3400](https://github.com/pipecat-ai/pipecat/pull/3400)) +- Fixed parallel function calling when using Gemini thinking. + (PR [3420](https://github.com/pipecat-ai/pipecat/pull/3420)) + - Fixed an issue in `traced_llm` where `model_name` in OpenTelemetry appears as `unknown`. (PR [#3422](https://github.com/pipecat-ai/pipecat/pull/3422)) diff --git a/README.md b/README.md index 0fbd1faead..65a12634f5 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,9 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout | Category | Services | | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Hathora](https://docs.pipecat.ai/server/services/stt/hathora), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) | -| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | +| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hathora](https://docs.pipecat.ai/server/services/tts/hathora), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | | Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) | diff --git a/changelog/3287.changed.md b/changelog/3287.changed.md deleted file mode 100644 index f0df829661..0000000000 --- a/changelog/3287.changed.md +++ /dev/null @@ -1 +0,0 @@ -- Enhanced interruption handling in `AsyncAITTSService` by supporting multi-context WebSocket sessions for more robust context management. \ No newline at end of file diff --git a/changelog/3287.fixed.md b/changelog/3287.fixed.md deleted file mode 100644 index 30ce0b13bc..0000000000 --- a/changelog/3287.fixed.md +++ /dev/null @@ -1 +0,0 @@ -- Corrected TTFB metric calculation in `AsyncAIHttpTTSService`. \ No newline at end of file diff --git a/docs/api/conf.py b/docs/api/conf.py index 6537276680..5f7a3c2dca 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -91,6 +91,25 @@ # MLX dependencies (Apple Silicon specific) "mlx", "mlx_whisper", # Note: might need underscore format too + # Pydantic v2 compatibility issues in third-party SDKs + "hume", + "hume.tts", + "hume.tts.types", + "cartesia", + "camb", + "sarvamai", + "openpipe", + "openai.types.beta.realtime", + "langchain_core", + "langchain_core.messages", + # FastAPI - Pydantic v2 compatibility issues during Sphinx autodoc + "fastapi", + "fastapi.applications", + "fastapi.routing", + "fastapi.params", + "fastapi.middleware", + "fastapi.responses", + "uvicorn", ] # HTML output settings diff --git a/env.example b/env.example index 1110a1ed33..c7b734ad6d 100644 --- a/env.example +++ b/env.example @@ -31,6 +31,9 @@ AZURE_DALLE_API_KEY=... AZURE_DALLE_ENDPOINT=https://... AZURE_DALLE_MODEL=... +# Camb.ai +CAMB_API_KEY=... + # Cartesia CARTESIA_API_KEY=... CARTESIA_VOICE_ID=... @@ -82,6 +85,9 @@ GROK_API_KEY=... # Groq GROQ_API_KEY=... +# Hathora +HATHORA_API_KEY=... + # Heygen HEYGEN_API_KEY=... HEYGEN_LIVE_AVATAR_API_KEY=... diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index 43618f84fb..e616543a31 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -10,7 +10,6 @@ from dotenv import load_dotenv from loguru import logger -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams diff --git a/examples/foundational/07n-interruptible-gemini-image.py b/examples/foundational/07n-interruptible-gemini-image.py index a465724f25..f32aebee16 100644 --- a/examples/foundational/07n-interruptible-gemini-image.py +++ b/examples/foundational/07n-interruptible-gemini-image.py @@ -45,7 +45,6 @@ from pipecat.transcriptions.language import Language from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams from pipecat.turns.user_stop import TurnAnalyzerUserTurnStopStrategy from pipecat.turns.user_turn_strategies import UserTurnStrategies diff --git a/examples/foundational/07p-interruptible-krisp-viva.py b/examples/foundational/07p-interruptible-krisp-viva.py index be336f003f..fd899f213e 100644 --- a/examples/foundational/07p-interruptible-krisp-viva.py +++ b/examples/foundational/07p-interruptible-krisp-viva.py @@ -28,7 +28,7 @@ from loguru import logger from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter -from pipecat.audio.turn.krisp_viva_turn import KrispTurnParams, KrispVivaTurn +from pipecat.audio.turn.krisp_viva_turn import KrispVivaTurn from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame diff --git a/examples/foundational/07aa-interruptible-soniox.py b/examples/foundational/07za-interruptible-soniox.py similarity index 100% rename from examples/foundational/07aa-interruptible-soniox.py rename to examples/foundational/07za-interruptible-soniox.py diff --git a/examples/foundational/07ab-interruptible-inworld-http.py b/examples/foundational/07zb-interruptible-inworld-http.py similarity index 100% rename from examples/foundational/07ab-interruptible-inworld-http.py rename to examples/foundational/07zb-interruptible-inworld-http.py diff --git a/examples/foundational/07ab-interruptible-inworld.py b/examples/foundational/07zb-interruptible-inworld.py similarity index 100% rename from examples/foundational/07ab-interruptible-inworld.py rename to examples/foundational/07zb-interruptible-inworld.py diff --git a/examples/foundational/07ac-interruptible-asyncai-http.py b/examples/foundational/07zc-interruptible-asyncai-http.py similarity index 100% rename from examples/foundational/07ac-interruptible-asyncai-http.py rename to examples/foundational/07zc-interruptible-asyncai-http.py diff --git a/examples/foundational/07ac-interruptible-asyncai.py b/examples/foundational/07zc-interruptible-asyncai.py similarity index 100% rename from examples/foundational/07ac-interruptible-asyncai.py rename to examples/foundational/07zc-interruptible-asyncai.py diff --git a/examples/foundational/07ad-interruptible-aicoustics.py b/examples/foundational/07zd-interruptible-aicoustics.py similarity index 100% rename from examples/foundational/07ad-interruptible-aicoustics.py rename to examples/foundational/07zd-interruptible-aicoustics.py diff --git a/examples/foundational/07ae-interruptible-hume.py b/examples/foundational/07ze-interruptible-hume.py similarity index 100% rename from examples/foundational/07ae-interruptible-hume.py rename to examples/foundational/07ze-interruptible-hume.py diff --git a/examples/foundational/07af-interruptible-gradium.py b/examples/foundational/07zf-interruptible-gradium.py similarity index 100% rename from examples/foundational/07af-interruptible-gradium.py rename to examples/foundational/07zf-interruptible-gradium.py diff --git a/examples/foundational/07zg-interruptible-camb.py b/examples/foundational/07zg-interruptible-camb.py new file mode 100644 index 0000000000..256eff6f7e --- /dev/null +++ b/examples/foundational/07zg-interruptible-camb.py @@ -0,0 +1,138 @@ +# +# Copyright (c) 2024–2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.camb.tts import CambTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.turns.user_stop import TurnAnalyzerUserTurnStopStrategy +from pipecat.turns.user_turn_strategies import UserTurnStrategies + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info("Starting Camb AI TTS bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CambTTSService( + api_key=os.getenv("CAMB_API_KEY"), + model="mars-flash", + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful voice assistant powered by Camb AI text-to-speech. " + "Keep your responses concise and conversational since they will be spoken aloud. " + "Avoid special characters, emojis, or bullet points.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + user_turn_strategies=UserTurnStrategies( + stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())] + ), + ), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + audio_out_sample_rate=22050, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info("Client connected") + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info("Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/07zh-interruptible-hathora.py similarity index 55% rename from examples/foundational/24-stt-mute-filter.py rename to examples/foundational/07zh-interruptible-hathora.py index 0c51716b88..f4bd169b1a 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/07zh-interruptible-hathora.py @@ -1,18 +1,14 @@ # -# Copyright (c) 2024-2026, Daily +# Copyright (c) 2024–2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # - -import asyncio import os from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams @@ -25,12 +21,10 @@ LLMContextAggregatorPair, LLMUserAggregatorParams, ) -from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFilter, STTMuteStrategy from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.deepgram.tts import DeepgramTTSService -from pipecat.services.llm_service import FunctionCallParams +from pipecat.services.hathora.stt import HathoraSTTService +from pipecat.services.hathora.tts import HathoraTTSService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -40,15 +34,6 @@ load_dotenv(override=True) - -async def fetch_weather_from_api(params: FunctionCallParams): - # Add a delay to test interruption during function calls - logger.info("Weather API call starting...") - await asyncio.sleep(5) # 5-second delay - logger.info("Weather API call completed") - await params.result_callback({"conditions": "nice", "temperature": "75"}) - - # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -74,50 +59,30 @@ async def fetch_weather_from_api(params: FunctionCallParams): async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + stt = HathoraSTTService( + model="nvidia-parakeet-tdt-0.6b-v3", + ) - # Configure the mute processor with both strategies - stt_mute_processor = STTMuteFilter( - config=STTMuteConfig( - strategies={ - STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE, - STTMuteStrategy.FUNCTION_CALL, - } - ), + tts = HathoraTTSService( + model="hexgrad-kokoro-82m", ) - tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") - - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - llm.register_function("get_current_weather", fetch_weather_from_api) - - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location", "format"], + # See https://models.hathora.dev/model/qwen3-30b-a3b + llm = OpenAILLMService( + base_url="https://app-362f7ca1-6975-4e18-a605-ab202bf2c315.app.hathora.dev/v1", + api_key=os.getenv("HATHORA_API_KEY"), + model=None, ) - tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", - "content": "You are a helpful assistant who can check the weather. Always check the weather when a location is mentioned. Respond concisely and naturally. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", }, ] - context = LLMContext(messages, tools) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams( user_turn_strategies=UserTurnStrategies( @@ -129,13 +94,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): pipeline = Pipeline( [ transport.input(), # Transport user input - stt, # STT - stt_mute_processor, # Add the mute processor between STT and context aggregator - user_aggregator, # User responses + stt, + context_aggregator.user(), # User responses llm, # LLM tts, # TTS transport.output(), # Transport bot output - assistant_aggregator, # Assistant spoken responses + context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -151,13 +115,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") - # Kick off the conversation with a weather-related prompt - messages.append( - { - "role": "system", - "content": "Ask the user what city they'd like to know the weather for.", - } - ) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index 8201790ac1..b517cf4476 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -22,7 +22,7 @@ ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index f1fe0d0683..52277a584b 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -13,7 +13,12 @@ from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import EndFrame, LLMMessagesAppendFrame, LLMRunFrame, TTSSpeakFrame +from pipecat.frames.frames import ( + EndTaskFrame, + LLMMessagesAppendFrame, + LLMRunFrame, + TTSSpeakFrame, +) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -22,7 +27,7 @@ LLMContextAggregatorPair, LLMUserAggregatorParams, ) -from pipecat.processors.user_idle_processor import UserIdleProcessor +from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService @@ -36,6 +41,43 @@ load_dotenv(override=True) + +class IdleHandler: + """Helper class to manage user idle retry logic.""" + + def __init__(self): + self._retry_count = 0 + + def reset(self): + """Reset the retry count when user becomes active.""" + self._retry_count = 0 + + async def handle_idle(self, aggregator): + """Handle user idle event with escalating prompts.""" + self._retry_count += 1 + + if self._retry_count == 1: + # First attempt: Add a gentle prompt to the conversation + message = { + "role": "system", + "content": "The user has been quiet. Politely and briefly ask if they're still there.", + } + await aggregator.push_frame(LLMMessagesAppendFrame([message], run_llm=True)) + elif self._retry_count == 2: + # Second attempt: More direct prompt + message = { + "role": "system", + "content": "The user is still inactive. Ask if they'd like to continue our conversation.", + } + await aggregator.push_frame(LLMMessagesAppendFrame([message], run_llm=True)) + else: + # Third attempt: End the conversation + await aggregator.push_frame( + TTSSpeakFrame("It seems like you're busy right now. Have a nice day!") + ) + await aggregator.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) + + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -84,42 +126,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): user_turn_strategies=UserTurnStrategies( stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())] ), + user_idle_timeout=5.0, # Detect user idle after 5 seconds ), ) - async def handle_user_idle(user_idle: UserIdleProcessor, retry_count: int) -> bool: - if retry_count == 1: - # First attempt: Add a gentle prompt to the conversation - message = { - "role": "system", - "content": "The user has been quiet. Politely and briefly ask if they're still there.", - } - await user_idle.push_frame(LLMMessagesAppendFrame([message], run_llm=True)) - return True - elif retry_count == 2: - # Second attempt: More direct prompt - message = { - "role": "system", - "content": "The user is still inactive. Ask if they'd like to continue our conversation.", - } - await user_idle.push_frame(LLMMessagesAppendFrame([message], run_llm=True)) - return True - else: - # Third attempt: End the conversation - await user_idle.push_frame( - TTSSpeakFrame("It seems like you're busy right now. Have a nice day!") - ) - await task.queue_frame(EndFrame()) - return False - - user_idle = UserIdleProcessor(callback=handle_user_idle, timeout=5.0) - pipeline = Pipeline( [ transport.input(), # Transport user input stt, - user_idle, # Idle user check-in - user_aggregator, + user_aggregator, # User aggregator with built-in idle detection llm, # LLM tts, # TTS transport.output(), # Transport bot output @@ -136,6 +151,17 @@ async def handle_user_idle(user_idle: UserIdleProcessor, retry_count: int) -> bo idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, ) + # Set up idle handling with retry logic + idle_handler = IdleHandler() + + @user_aggregator.event_handler("on_user_turn_idle") + async def on_user_turn_idle(aggregator): + await idle_handler.handle_idle(aggregator) + + @user_aggregator.event_handler("on_user_turn_started") + async def on_user_turn_started(aggregator, strategy): + idle_handler.reset() + @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index a29eb5cf96..e0e2949a3d 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -17,7 +17,7 @@ from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, TTSSpeakFrame, UserImageRequestFrame +from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index bd95fb7e73..10101f191f 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -22,7 +22,6 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService @@ -114,6 +113,14 @@ async def _reset(): # "content": f"{AWSNovaSonicLLMService.AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION}", # } # ) + # If the last message isn't from the user, add a message asking for a recap + if messages and messages[-1].get("role") != "user": + messages.append( + { + "role": "user", + "content": "Can you catch me up on what we were talking about?", + } + ) params.context.set_messages(messages) await params.llm.reset_conversation() # await params.llm.trigger_assistant_response() diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 256a6c9dfd..9a6d224940 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -119,7 +119,7 @@ async def process_frame(self, frame: Frame, direction: FrameDirection): if isinstance(frame, TextFrame) and frame.text == "YES": logger.debug("Completeness check YES") - await self.push_frame(UserStoppedSpeakingFrame()) + await self.broadcast_frame(UserStoppedSpeakingFrame) await self._notifier.notify() elif isinstance(frame, TextFrame) and frame.text == "NO": logger.debug("Completeness check NO") diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index bdb5575941..935aa5e361 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -322,7 +322,7 @@ async def process_frame(self, frame: Frame, direction: FrameDirection): if isinstance(frame, TextFrame) and frame.text == "YES": logger.debug("!!! Completeness check YES") - await self.push_frame(UserStoppedSpeakingFrame()) + await self.broadcast_frame(UserStoppedSpeakingFrame) await self._notifier.notify() elif isinstance(frame, TextFrame) and frame.text == "NO": logger.debug("!!! Completeness check NO") diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index b144f485ec..4a7c33a39b 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -451,7 +451,7 @@ async def process_frame(self, frame: Frame, direction: FrameDirection): logger.debug("Completeness check YES") if self._idle_task: await self.cancel_task(self._idle_task) - await self.push_frame(UserStoppedSpeakingFrame()) + await self.broadcast_frame(UserStoppedSpeakingFrame) await self._audio_accumulator.reset() await self._notifier.notify() elif isinstance(frame, TextFrame): diff --git a/examples/foundational/24-user-mute-strategy.py b/examples/foundational/24-user-mute-strategy.py index df6619fd40..0797fc092f 100644 --- a/examples/foundational/24-user-mute-strategy.py +++ b/examples/foundational/24-user-mute-strategy.py @@ -34,7 +34,7 @@ from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams -from pipecat.turns.mute import ( +from pipecat.turns.user_mute import ( FunctionCallUserMuteStrategy, MuteUntilFirstBotCompleteUserMuteStrategy, ) @@ -161,6 +161,14 @@ async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") await task.cancel() + @user_aggregator.event_handler("on_user_mute_started") + async def on_user_mute_started(aggregator): + logger.info(f"User mute started") + + @user_aggregator.event_handler("on_user_mute_stopped") + async def on_user_mute_stopped(aggregator): + logger.info(f"User mute stopped") + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) await runner.run(task) diff --git a/examples/foundational/26e-gemini-live-google-search.py b/examples/foundational/26e-gemini-live-google-search.py index 294cf1789b..9b37f32ee0 100644 --- a/examples/foundational/26e-gemini-live-google-search.py +++ b/examples/foundational/26e-gemini-live-google-search.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024-2025, Daily +# Copyright (c) 2024-2026, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index ff43692245..7647287ed5 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024-2025, Daily +# Copyright (c) 2024-2026, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/39c-multiple-mcp.py b/examples/foundational/39c-multiple-mcp.py index 6141b371e9..05cbe2b221 100644 --- a/examples/foundational/39c-multiple-mcp.py +++ b/examples/foundational/39c-multiple-mcp.py @@ -9,7 +9,6 @@ import io import json import os -import re import shutil import aiohttp diff --git a/examples/foundational/49d-thinking-functions-google.py b/examples/foundational/49d-thinking-functions-google.py index 1a74da3ffe..e00c3d00f7 100644 --- a/examples/foundational/49d-thinking-functions-google.py +++ b/examples/foundational/49d-thinking-functions-google.py @@ -13,7 +13,7 @@ from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame, ThoughtTranscriptionMessage, TranscriptionMessage +from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask diff --git a/examples/foundational/51-grok-realtime.py b/examples/foundational/51-grok-realtime.py index 6a2fbe5a27..87486a0de2 100644 --- a/examples/foundational/51-grok-realtime.py +++ b/examples/foundational/51-grok-realtime.py @@ -53,8 +53,6 @@ from pipecat.runner.utils import create_transport from pipecat.services.grok.realtime.events import ( SessionProperties, - WebSearchTool, - XSearchTool, ) from pipecat.services.grok.realtime.llm import GrokRealtimeLLMService from pipecat.services.llm_service import FunctionCallParams diff --git a/examples/quickstart/pcc-deploy.toml b/examples/quickstart/pcc-deploy.toml index ff77c45dc8..812e2b8e66 100644 --- a/examples/quickstart/pcc-deploy.toml +++ b/examples/quickstart/pcc-deploy.toml @@ -1,11 +1,11 @@ -agent_name = "quickstart" -image = "your_username/quickstart:0.1" -secret_set = "quickstart-secrets" +agent_name = "quickstart-test" +image = "markatdaily/quickstart-test:latest" +secret_set = "quickstart-test-secrets" agent_profile = "agent-1x" # RECOMMENDED: Set an image pull secret: # https://docs.pipecat.ai/deployment/pipecat-cloud/fundamentals/secrets#image-pull-secrets -# image_credentials = "your_image_pull_secret" +image_credentials = "dockerhub-access" [scaling] min_agents = 1 diff --git a/pyproject.toml b/pyproject.toml index 7f50189de7..e99ab62bc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,11 @@ dependencies = [ ] [project.urls] +Homepage = "https://pipecat.ai" +Documentation = "https://docs.pipecat.ai/" Source = "https://github.com/pipecat-ai/pipecat" -Website = "https://pipecat.ai" +Issues = "https://github.com/pipecat-ai/pipecat/issues" +Changelog = "https://github.com/pipecat-ai/pipecat/blob/main/CHANGELOG.md" [project.optional-dependencies] aic = [ "aic-sdk~=1.2.0" ] @@ -53,6 +56,7 @@ aws = [ "aioboto3~=15.5.0", "pipecat-ai[websockets-base]" ] aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.2.0; python_version>='3.12'" ] azure = [ "azure-cognitiveservices-speech~=1.44.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] +camb = [ "camb-sdk>=1.5.4" ] cerebras = [] daily = [ "daily-python~=0.23.0" ] deepgram = [ "deepgram-sdk~=4.7.0", "pipecat-ai[websockets-base]" ] @@ -96,7 +100,7 @@ qwen = [] remote-smart-turn = [] rime = [ "pipecat-ai[websockets-base]" ] riva = [ "pipecat-ai[nvidia]" ] -runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=2.0.4"] +runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.128.0", "pipecat-ai-small-webrtc-prebuilt>=2.0.4"] sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] sambanova = [] sarvam = [ "sarvamai==0.1.21", "pipecat-ai[websockets-base]" ] @@ -112,7 +116,7 @@ together = [] tracing = [ "opentelemetry-sdk>=1.33.0", "opentelemetry-api>=1.33.0", "opentelemetry-instrumentation>=0.54b0" ] ultravox = [ "pipecat-ai[websockets-base]" ] webrtc = [ "aiortc>=1.14.0,<2", "opencv-python>=4.11.0.86,<5" ] -websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<0.122.0" ] +websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<0.128.0" ] websockets-base = [ "websockets>=13.1,<16.0" ] whisper = [ "faster-whisper~=1.1.1" ] diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index 7c6a726043..16598b4d7e 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -293,12 +293,13 @@ async def run_eval_pipeline( "You should only call the eval function if:\n" "- The user explicitly attempts to answer the question, AND\n" f"- Their answer can be cleanly evaluated using: {eval_config.eval}\n" - "Ignore greetings, comments, non-answers, or requests for clarification." + "Ignore greetings, comments, non-answers, or requests for clarification.\n" + "Numerical word answers are allowed (e.g., 'five' is the same as '5').\n" ) if eval_config.eval_speaks_first: - system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" + system_prompt = f"You are an evaluation agent, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" else: - system_prompt = f"You are an evaluation agent, be extremly brief. Numerical word answers are allowed. First, ask one question: {example_prompt}. {common_system_prompt}" + system_prompt = f"You are an evaluation agent, be extremly brief. First, ask one question: {example_prompt}. {common_system_prompt}" messages = [ { diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index dac7dd401f..e544c732ea 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -97,15 +97,6 @@ def EVAL_VISION_IMAGE(*, eval_speaks_first: bool = False): ("07-interruptible-cartesia-http.py", EVAL_SIMPLE_MATH), ("07a-interruptible-speechmatics.py", EVAL_SIMPLE_MATH), ("07a-interruptible-speechmatics-vad.py", EVAL_SIMPLE_MATH), - ("07aa-interruptible-soniox.py", EVAL_SIMPLE_MATH), - ("07ab-interruptible-inworld.py", EVAL_SIMPLE_MATH), - ("07ab-interruptible-inworld-http.py", EVAL_SIMPLE_MATH), - ("07ac-interruptible-asyncai.py", EVAL_SIMPLE_MATH), - ("07ac-interruptible-asyncai-http.py", EVAL_SIMPLE_MATH), - # Need license key to run - # ("07ad-interruptible-aicoustics.py", EVAL_SIMPLE_MATH), - ("07ae-interruptible-hume.py", EVAL_SIMPLE_MATH), - ("07af-interruptible-gradium.py", EVAL_SIMPLE_MATH), ("07b-interruptible-langchain.py", EVAL_SIMPLE_MATH), ("07c-interruptible-deepgram.py", EVAL_SIMPLE_MATH), ("07c-interruptible-deepgram-flux.py", EVAL_SIMPLE_MATH), @@ -137,6 +128,17 @@ def EVAL_VISION_IMAGE(*, eval_speaks_first: bool = False): ("07y-interruptible-minimax.py", EVAL_SIMPLE_MATH), ("07z-interruptible-sarvam.py", EVAL_SIMPLE_MATH), ("07z-interruptible-sarvam-http.py", EVAL_SIMPLE_MATH), + ("07za-interruptible-soniox.py", EVAL_SIMPLE_MATH), + ("07zb-interruptible-inworld.py", EVAL_SIMPLE_MATH), + ("07zb-interruptible-inworld-http.py", EVAL_SIMPLE_MATH), + ("07zc-interruptible-asyncai.py", EVAL_SIMPLE_MATH), + ("07zc-interruptible-asyncai-http.py", EVAL_SIMPLE_MATH), + # Need license key to run + # ("07zd-interruptible-aicoustics.py", EVAL_SIMPLE_MATH), + ("07ze-interruptible-hume.py", EVAL_SIMPLE_MATH), + ("07zf-interruptible-gradium.py", EVAL_SIMPLE_MATH), + ("07zg-interruptible-camb.py", EVAL_SIMPLE_MATH), + ("07zh-interruptible-hathora.py", EVAL_SIMPLE_MATH), # Needs a local XTTS docker instance running. # ("07i-interruptible-xtts.py", EVAL_SIMPLE_MATH), # Needs a Krisp license. diff --git a/scripts/krisp/test_krisp_viva_filter_audiofile.py b/scripts/krisp/test_krisp_viva_filter_audiofile.py index c75dc19fc5..65e9d7685e 100644 --- a/scripts/krisp/test_krisp_viva_filter_audiofile.py +++ b/scripts/krisp/test_krisp_viva_filter_audiofile.py @@ -22,7 +22,7 @@ try: import numpy as np - import soundfile as sf + import soundfile as sf # noqa: F401 from audio_file_utils import calculate_audio_stats, read_audio_file, write_audio_file except ImportError as e: print(f"Error: Missing required dependencies: {e}") diff --git a/scripts/krisp/test_krisp_viva_turn_audiofile.py b/scripts/krisp/test_krisp_viva_turn_audiofile.py index c380ad98fb..6580e8e906 100644 --- a/scripts/krisp/test_krisp_viva_turn_audiofile.py +++ b/scripts/krisp/test_krisp_viva_turn_audiofile.py @@ -23,7 +23,7 @@ try: import numpy as np - import soundfile as sf + import soundfile as sf # noqa: F401 from audio_file_utils import read_audio_file except ImportError as e: print(f"Error: Missing required dependencies: {e}") diff --git a/src/pipecat/adapters/services/bedrock_adapter.py b/src/pipecat/adapters/services/bedrock_adapter.py index 8e46b18990..ccbbe5e2e4 100644 --- a/src/pipecat/adapters/services/bedrock_adapter.py +++ b/src/pipecat/adapters/services/bedrock_adapter.py @@ -10,7 +10,7 @@ import copy import json from dataclasses import dataclass -from typing import Any, Dict, List, Literal, Optional, TypedDict +from typing import Any, Dict, List, Optional, TypedDict from loguru import logger diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 2de3742c83..4968c2719c 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -9,7 +9,7 @@ import base64 import json from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Tuple, TypedDict +from typing import Any, Dict, List, Optional, TypedDict from loguru import logger from openai import NotGiven diff --git a/src/pipecat/adapters/services/open_ai_adapter.py b/src/pipecat/adapters/services/open_ai_adapter.py index 6c44a3404e..f4b534f2cc 100644 --- a/src/pipecat/adapters/services/open_ai_adapter.py +++ b/src/pipecat/adapters/services/open_ai_adapter.py @@ -7,10 +7,8 @@ """OpenAI LLM adapter for Pipecat.""" import copy -import json from typing import Any, Dict, List, TypedDict -from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN from openai._types import NotGiven as OpenAINotGiven from openai.types.chat import ( ChatCompletionMessageParam, diff --git a/src/pipecat/audio/filters/krisp_filter.py b/src/pipecat/audio/filters/krisp_filter.py index 7fbc00b88e..58c4a28354 100644 --- a/src/pipecat/audio/filters/krisp_filter.py +++ b/src/pipecat/audio/filters/krisp_filter.py @@ -61,6 +61,7 @@ class KrispFilter(BaseAudioFilter): Provides real-time noise reduction for audio streams using Krisp's proprietary noise suppression algorithms. Requires a Krisp model file for operation. + .. deprecated:: 0.0.94 The KrispFilter is deprecated and will be removed in a future version. Use KrispVivaFilter instead. diff --git a/src/pipecat/audio/filters/krisp_viva_filter.py b/src/pipecat/audio/filters/krisp_viva_filter.py index 2f9dda10fd..ea5bfb8de2 100644 --- a/src/pipecat/audio/filters/krisp_viva_filter.py +++ b/src/pipecat/audio/filters/krisp_viva_filter.py @@ -9,7 +9,6 @@ This module provides an audio filter implementation using Krisp VIVA SDK. """ -import asyncio import os import numpy as np diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 81a92800a5..1b53e9934c 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -1024,10 +1024,8 @@ async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): logger.debug( f"{self} FunctionCallCancelFrame: [{frame.function_name}:{frame.tool_call_id}]" ) - if frame.tool_call_id not in self._function_calls_in_progress: - return - - if self._function_calls_in_progress[frame.tool_call_id].cancel_on_interruption: + function_call = self._function_calls_in_progress.get(frame.tool_call_id) + if function_call and function_call.cancel_on_interruption: await self.handle_function_call_cancel(frame) del self._function_calls_in_progress[frame.tool_call_id] diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 2854c86815..81dbfe8440 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -62,7 +62,8 @@ NotGiven, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.turns.mute import BaseUserMuteStrategy +from pipecat.turns.user_idle_controller import UserIdleController +from pipecat.turns.user_mute import BaseUserMuteStrategy from pipecat.turns.user_start import BaseUserTurnStartStrategy, UserTurnStartedParams from pipecat.turns.user_stop import BaseUserTurnStopStrategy, UserTurnStoppedParams from pipecat.turns.user_turn_controller import UserTurnController @@ -80,11 +81,16 @@ class LLMUserAggregatorParams: user_mute_strategies: List of user mute strategies. user_turn_stop_timeout: Time in seconds to wait before considering the user's turn finished. + user_idle_timeout: Optional timeout in seconds for detecting user idle state. + If set, the aggregator will emit an `on_user_turn_idle` event when the user + has been idle (not speaking) for this duration. Set to None to disable + idle detection. """ user_turn_strategies: Optional[UserTurnStrategies] = None user_mute_strategies: List[BaseUserMuteStrategy] = field(default_factory=list) user_turn_stop_timeout: float = 5.0 + user_idle_timeout: Optional[float] = None @dataclass @@ -291,11 +297,14 @@ class LLMUserAggregator(LLMContextAggregator): - on_user_turn_started: Called when the user turn starts - on_user_turn_stopped: Called when the user turn ends - on_user_turn_stop_timeout: Called when no user turn stop strategy triggers + - on_user_turn_idle: Called when the user has been idle for the configured timeout + - on_user_mute_started: Called when the user becomes muted + - on_user_mute_stopped: Called when the user becomes unmuted Example:: @aggregator.event_handler("on_user_turn_started") - async def on_user_turn_started(aggregator, strategy: BaseUserTurnStartStrategy]): + async def on_user_turn_started(aggregator, strategy: BaseUserTurnStartStrategy): ... @aggregator.event_handler("on_user_turn_stopped") @@ -306,6 +315,18 @@ async def on_user_turn_stopped(aggregator, strategy: BaseUserTurnStopStrategy, m async def on_user_turn_stop_timeout(aggregator): ... + @aggregator.event_handler("on_user_turn_idle") + async def on_user_turn_idle(aggregator): + ... + + @aggregator.event_handler("on_user_mute_started") + async def on_user_mute_started(aggregator): + ... + + @aggregator.event_handler("on_user_mute_stopped") + async def on_user_mute_stopped(aggregator): + ... + """ def __init__( @@ -328,6 +349,9 @@ def __init__( self._register_event_handler("on_user_turn_started") self._register_event_handler("on_user_turn_stopped") self._register_event_handler("on_user_turn_stop_timeout") + self._register_event_handler("on_user_turn_idle") + self._register_event_handler("on_user_mute_started") + self._register_event_handler("on_user_mute_stopped") user_turn_strategies = self._params.user_turn_strategies or UserTurnStrategies() @@ -350,6 +374,16 @@ def __init__( "on_user_turn_stop_timeout", self._on_user_turn_stop_timeout ) + # Optional user idle controller + self._user_idle_controller: Optional[UserIdleController] = None + if self._params.user_idle_timeout: + self._user_idle_controller = UserIdleController( + user_idle_timeout=self._params.user_idle_timeout + ) + self._user_idle_controller.add_event_handler( + "on_user_turn_idle", self._on_user_turn_idle + ) + async def cleanup(self): """Clean up processor resources.""" await super().cleanup() @@ -405,6 +439,9 @@ async def process_frame(self, frame: Frame, direction: FrameDirection): await self._user_turn_controller.process_frame(frame) + if self._user_idle_controller: + await self._user_idle_controller.process_frame(frame) + async def push_aggregation(self) -> str: """Push the current aggregation.""" if len(self._aggregation) == 0: @@ -420,6 +457,9 @@ async def push_aggregation(self) -> str: async def _start(self, frame: StartFrame): await self._user_turn_controller.setup(self.task_manager) + if self._user_idle_controller: + await self._user_idle_controller.setup(self.task_manager) + for s in self._params.user_mute_strategies: await s.setup(self.task_manager) @@ -432,6 +472,9 @@ async def _cancel(self, frame: CancelFrame): async def _cleanup(self): await self._user_turn_controller.cleanup() + if self._user_idle_controller: + await self._user_idle_controller.cleanup() + for s in self._params.user_mute_strategies: await s.cleanup() @@ -461,6 +504,12 @@ async def _maybe_mute_frame(self, frame: Frame): logger.debug(f"{self}: user is now {'muted' if should_mute_next_time else 'unmuted'}") self._user_is_muted = should_mute_next_time + # Emit mute state change events + if self._user_is_muted: + await self._call_event_handler("on_user_mute_started") + else: + await self._call_event_handler("on_user_mute_stopped") + return should_mute_frame async def _handle_llm_run(self, frame: LLMRunFrame): @@ -565,6 +614,9 @@ async def _on_user_turn_stopped( async def _on_user_turn_stop_timeout(self, controller): await self._call_event_handler("on_user_turn_stop_timeout") + async def _on_user_turn_idle(self, controller): + await self._call_event_handler("on_user_turn_idle") + class LLMAssistantAggregator(LLMContextAggregator): """Assistant LLM aggregator that processes bot responses and function calls. @@ -858,10 +910,8 @@ async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): logger.debug( f"{self} FunctionCallCancelFrame: [{frame.function_name}:{frame.tool_call_id}]" ) - if frame.tool_call_id not in self._function_calls_in_progress: - return - - if self._function_calls_in_progress[frame.tool_call_id].cancel_on_interruption: + function_call = self._function_calls_in_progress.get(frame.tool_call_id) + if function_call and function_call.cancel_on_interruption: # Update context with the function call cancellation self._update_function_call_result(frame.function_name, frame.tool_call_id, "CANCELLED") del self._function_calls_in_progress[frame.tool_call_id] diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 41df3b5e8b..f756251560 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -34,7 +34,6 @@ from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.frames.frames import AudioRawFrame, Frame -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor # JSON custom encoder to handle bytes arrays so that we can log contexts # with images to the console. diff --git a/src/pipecat/processors/filters/wake_check_filter.py b/src/pipecat/processors/filters/wake_check_filter.py index 792c4a68d3..ec8f31f530 100644 --- a/src/pipecat/processors/filters/wake_check_filter.py +++ b/src/pipecat/processors/filters/wake_check_filter.py @@ -18,7 +18,7 @@ from loguru import logger -from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 9c26fe3825..70f44dfca5 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -950,7 +950,8 @@ def __reset_process_queue(self): # Process current queue and keep UninterruptibleFrame frames. while not self.__process_queue.empty(): item = self.__process_queue.get_nowait() - if isinstance(item, UninterruptibleFrame): + frame = item[0] + if isinstance(frame, UninterruptibleFrame): new_queue.put_nowait(item) self.__process_queue.task_done() diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index 6dc6dd47c3..67c41ab131 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -8,6 +8,7 @@ import asyncio import inspect +import warnings from typing import Awaitable, Callable, Union from pipecat.frames.frames import ( @@ -26,6 +27,10 @@ class UserIdleProcessor(FrameProcessor): """Monitors user inactivity and triggers callbacks after timeout periods. + .. deprecated:: + UserIdleProcessor is deprecated in 0.0.100 and will be removed in a future version. + Use LLMUserAggregator with user_idle_timeout parameter instead. + This processor tracks user activity and triggers configurable callbacks when users become idle. It starts monitoring only after the first conversation activity and supports both basic and retry-based callback patterns. @@ -70,6 +75,14 @@ def __init__( **kwargs: Additional arguments passed to FrameProcessor. """ super().__init__(**kwargs) + + warnings.warn( + "UserIdleProcessor is deprecated in 0.0.100 and will be removed in a " + "future version. Use LLMUserAggregator with user_idle_timeout parameter " + "instead.", + DeprecationWarning, + ) + self._callback = self._wrap_callback(callback) self._timeout = timeout self._retry_count = 0 diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index 0396e393bf..96fbf2598c 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -263,7 +263,7 @@ async def offer(request: SmallWebRTCRequest, background_tasks: BackgroundTasks): """Handle WebRTC offer requests via SmallWebRTCRequestHandler.""" # Prepare runner arguments with the callback to run your bot - async def webrtc_connection_callback(connection): + async def webrtc_connection_callback(connection: SmallWebRTCConnection): bot_module = _get_bot_module() runner_args = SmallWebRTCRunnerArguments( @@ -406,13 +406,7 @@ def _setup_whatsapp_routes(app: FastAPI): return try: - from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI - from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection - from pipecat.transports.smallwebrtc.request_handler import ( - SmallWebRTCRequest, - SmallWebRTCRequestHandler, - ) from pipecat.transports.whatsapp.api import WhatsAppWebhookRequest from pipecat.transports.whatsapp.client import WhatsAppClient except ImportError as e: diff --git a/src/pipecat/serializers/vonage.py b/src/pipecat/serializers/vonage.py index 9de1cc0380..3213e8bbae 100644 --- a/src/pipecat/serializers/vonage.py +++ b/src/pipecat/serializers/vonage.py @@ -34,8 +34,7 @@ class VonageFrameSerializer(FrameSerializer): WebSocket streaming protocol. Note: - Ref docs: - https://developer.vonage.com/en/video/guides/audio-connector + Ref docs: https://developer.vonage.com/en/video/guides/audio-connector """ class InputParams(BaseModel): diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index a9952fa000..c03ab9d0e4 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -148,11 +148,11 @@ async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, StartFrame): - await self.start(frame) - elif isinstance(frame, CancelFrame): - await self.cancel(frame) + await self._start(frame) elif isinstance(frame, EndFrame): - await self.stop(frame) + await self._stop(frame) + elif isinstance(frame, CancelFrame): + await self._cancel(frame) async def process_generator(self, generator: AsyncGenerator[Frame | None, None]): """Process frames from an async generator. @@ -169,3 +169,21 @@ async def process_generator(self, generator: AsyncGenerator[Frame | None, None]) await self.push_error_frame(f) else: await self.push_frame(f) + + async def _start(self, frame: StartFrame): + try: + await self.start(frame) + except Exception as e: + logger.error(f"{self}: exception processing {frame}: {e}") + + async def _stop(self, frame: EndFrame): + try: + await self.stop(frame) + except Exception as e: + logger.error(f"{self}: exception processing {frame}: {e}") + + async def _cancel(self, frame: CancelFrame): + try: + await self.cancel(frame) + except Exception as e: + logger.error(f"{self}: exception processing {frame}: {e}") diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 26afc79cf5..05baba2bdd 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -38,6 +38,7 @@ LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, + LLMTextFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, @@ -295,6 +296,7 @@ def __init__( self._user_text_buffer = "" self._assistant_text_buffer = "" self._completed_tool_calls = set() + self._audio_input_started = False file_path = files("pipecat.services.aws.nova_sonic").joinpath("ready.wav") with wave.open(file_path.open("rb"), "rb") as wav_file: @@ -531,14 +533,30 @@ async def _finish_connecting_if_context_available(self): if system_instruction: await self._send_text_event(text=system_instruction, role=Role.SYSTEM) - # Send conversation history - for message in llm_connection_params["messages"]: + # Send conversation history (except for the last message if it's from the + # user, which we'll send as interactive after starting audio input) + messages = llm_connection_params["messages"] + last_user_message = None + for i, message in enumerate(messages): # logger.debug(f"Seeding conversation history with message: {message}") - await self._send_text_event(text=message.text, role=message.role) + is_last_message = i == len(messages) - 1 + if is_last_message and message.role == Role.USER: + # Save for sending after audio input starts + last_user_message = message + else: + await self._send_text_event(text=message.text, role=message.role) # Start audio input await self._send_audio_input_start_event() + # Now send the last user message as interactive to trigger bot response + if last_user_message: + # logger.debug( + # f"Sending last user message as interactive to trigger bot response: {last_user_message}") + await self._send_text_event( + text=last_user_message.text, role=last_user_message.role, interactive=True + ) + # Start receiving events self._receive_task = self.create_task(self._receive_task_handler()) @@ -601,6 +619,7 @@ async def _disconnect(self): self._user_text_buffer = "" self._assistant_text_buffer = "" self._completed_tool_calls = set() + self._audio_input_started = False logger.info("Finished disconnecting") except Exception as e: @@ -726,8 +745,18 @@ async def _send_audio_input_start_event(self): }} ''' await self._send_client_event(audio_content_start) + self._audio_input_started = True + + async def _send_text_event(self, text: str, role: Role, interactive: bool = False): + """Send a text event to the LLM. - async def _send_text_event(self, text: str, role: Role): + Args: + text: The text content to send. + role: The role associated with the text (e.g., USER, ASSISTANT, SYSTEM). + interactive: Whether the content is interactive. Defaults to False. + False: conversation history or system instruction, sent prior to interactive audio + True: text input sent during (or at the start of) interactive audio + """ if not self._stream or not self._prompt_name or not text: return @@ -740,7 +769,7 @@ async def _send_text_event(self, text: str, role: Role): "promptName": "{self._prompt_name}", "contentName": "{content_name}", "type": "TEXT", - "interactive": true, + "interactive": {json.dumps(interactive)}, "role": "{role.value}", "textInputConfiguration": {{ "mediaType": "text/plain" @@ -778,7 +807,7 @@ async def _send_text_event(self, text: str, role: Role): await self._send_client_event(text_content_end) async def _send_user_audio_event(self, audio: bytes): - if not self._stream: + if not self._stream or not self._audio_input_started: return blob = base64.b64encode(audio) @@ -1077,9 +1106,7 @@ async def _report_assistant_response_text_added(self, text): logger.debug(f"Assistant response text added: {text}") # Report the text of the assistant response. - frame = TTSTextFrame(text, aggregated_by=AggregationType.SENTENCE) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self._push_assistant_response_text_frames(text) # HACK: here we're also buffering the assistant text ourselves as a # backup rather than relying solely on the assistant context aggregator @@ -1112,11 +1139,7 @@ async def _report_assistant_response_ended(self): # TTSTextFrame would be ignored otherwise (the interruption frame # would have cleared the assistant aggregator state). await self.push_frame(LLMFullResponseStartFrame()) - frame = TTSTextFrame( - self._assistant_text_buffer, aggregated_by=AggregationType.SENTENCE - ) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self._push_assistant_response_text_frames(self._assistant_text_buffer) self._may_need_repush_assistant_text = False # Report the end of the assistant response. @@ -1128,6 +1151,25 @@ async def _report_assistant_response_ended(self): # Clear out the buffered assistant text self._assistant_text_buffer = "" + async def _push_assistant_response_text_frames(self, text: str): + # In a typical "cascade" LLM + TTS setup, LLMTextFrames would not + # proceed beyond the TTS service. Therefore, since a speech-to-speech + # service like Nova Sonic combines both LLM and TTS functionality, you + # would think we wouldn't need to push LLMTextFrames at all. However, + # RTVI relies on LLMTextFrames being pushed to trigger its + # "bot-llm-text" event. So here we push an LLMTextFrame, too, but avoid + # appending it to context to avoid context message duplication. + + # Push LLMTextFrame + llm_text_frame = LLMTextFrame(text) + llm_text_frame.append_to_context = False + await self.push_frame(llm_text_frame) + + # Push TTSTextFrame + tts_text_frame = TTSTextFrame(text, aggregated_by=AggregationType.SENTENCE) + tts_text_frame.includes_inter_frame_spaces = True + await self.push_frame(tts_text_frame) + # # user transcription reporting # @@ -1187,7 +1229,7 @@ async def _report_user_transcription_ended(self): logger.debug( "Wrapping assistant response trigger transcription with upstream UserStarted/StoppedSpeakingFrames" ) - await self.push_frame(UserStartedSpeakingFrame(), direction=FrameDirection.UPSTREAM) + await self.broadcast_frame(UserStartedSpeakingFrame) # Send the transcription upstream for the user context aggregator frame = TranscriptionFrame( @@ -1197,7 +1239,7 @@ async def _report_user_transcription_ended(self): # Finish wrapping the upstream transcription in UserStarted/StoppedSpeakingFrames if needed if should_wrap_in_user_started_stopped_speaking_frames: - await self.push_frame(UserStoppedSpeakingFrame(), direction=FrameDirection.UPSTREAM) + await self.broadcast_frame(UserStoppedSpeakingFrame) # Clear out the buffered user text self._user_text_buffer = "" diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index 2ad350a960..5ae99b02f3 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -10,7 +10,6 @@ speech-to-text transcription with support for multiple languages and audio formats. """ -import asyncio import json import os import random diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index 0df2dabd9a..358a894c7c 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -10,7 +10,6 @@ supporting multiple languages, voices, and SSML features. """ -import asyncio import os from typing import AsyncGenerator, List, Optional diff --git a/src/pipecat/services/aws_nova_sonic/__init__.py b/src/pipecat/services/aws_nova_sonic/__init__.py index 8348e9ffec..a198094cbc 100644 --- a/src/pipecat/services/aws_nova_sonic/__init__.py +++ b/src/pipecat/services/aws_nova_sonic/__init__.py @@ -17,3 +17,8 @@ DeprecationWarning, stacklevel=2, ) + +__all__ = [ + "AWSNovaSonicLLMService", + "Params", +] diff --git a/src/pipecat/services/azure/common.py b/src/pipecat/services/azure/common.py index f867d4e5db..dc7aaa3594 100644 --- a/src/pipecat/services/azure/common.py +++ b/src/pipecat/services/azure/common.py @@ -8,8 +8,6 @@ from typing import Optional -from loguru import logger - from pipecat.transcriptions.language import Language, resolve_language diff --git a/src/pipecat/services/azure/image.py b/src/pipecat/services/azure/image.py index b33d8cc7df..2bddf6c439 100644 --- a/src/pipecat/services/azure/image.py +++ b/src/pipecat/services/azure/image.py @@ -15,7 +15,6 @@ from typing import AsyncGenerator import aiohttp -from loguru import logger from PIL import Image from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index 751320c195..93c421c1ee 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -277,6 +277,8 @@ def __init__( self._started = False self._first_chunk = True self._cumulative_audio_offset: float = 0.0 # Cumulative audio duration in seconds + self._last_word: Optional[str] = None # Track last word for punctuation merging + self._last_timestamp: Optional[float] = None # Track last timestamp def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -346,9 +348,34 @@ async def cancel(self, frame: CancelFrame): await self.cancel_task(self._word_processor_task) self._word_processor_task = None + def _is_cjk_language(self) -> bool: + """Check if the configured language is CJK (Chinese, Japanese, Korean). + + Returns: + True if the language is CJK, False otherwise. + """ + language = self._settings.get("language", "").lower() + # Check if language starts with CJK language codes + return language.startswith(("zh", "ja", "ko", "cmn", "yue", "wuu")) + + def _is_punctuation_only(self, text: str) -> bool: + """Check if text consists only of punctuation and whitespace. + + Args: + text: Text to check. + + Returns: + True if text is only punctuation/whitespace, False otherwise. + """ + return text and all(not c.isalnum() for c in text) + def _handle_word_boundary(self, evt): """Handle word boundary events from Azure SDK. + Azure sends punctuation as separate word boundaries, and breaks CJK text + into individual characters/particles. This method routes to language-specific + handlers to properly merge and emit word boundaries. + Args: evt: SpeechSynthesisWordBoundaryEventArgs from Azure Speech SDK containing word text and audio offset timing. @@ -362,13 +389,75 @@ def _handle_word_boundary(self, evt): # Add cumulative offset to get absolute timestamp across sentences absolute_seconds = self._cumulative_audio_offset + sentence_relative_seconds - # Queue word timestamp for async processing - # Use thread-safe queue since this is called from Azure SDK thread - if word: - logger.trace(f"{self}: Word boundary - '{word}' at {absolute_seconds:.2f}s") - # Put in temporary queue - will be processed by async task - # Store as (word, timestamp_in_seconds) tuple - self._word_boundary_queue.put_nowait((word, absolute_seconds)) + if not word: + return + + # Route to language-specific handler + if self._is_cjk_language(): + self._handle_cjk_word_boundary(word, absolute_seconds) + else: + self._handle_non_cjk_word_boundary(word, absolute_seconds) + + def _emit_pending_word(self): + """Emit the currently buffered word if one exists.""" + if self._last_word is not None: + self._word_boundary_queue.put_nowait((self._last_word, self._last_timestamp)) + self._last_word = None + self._last_timestamp = None + + def _handle_cjk_word_boundary(self, word: str, timestamp: float): + """Handle word boundaries for CJK languages (Chinese, Japanese, Korean). + + CJK languages don't use spaces between words, so we merge characters together + and only emit at natural break points (punctuation or whitespace boundaries). + Without this logic, we don't get word output for CJK languages. + + Args: + word: The word/character from Azure. + timestamp: Timestamp in seconds. + """ + # First word: just store it + if self._last_word is None: + self._last_word = word + self._last_timestamp = timestamp + return + + # Punctuation: merge and emit (natural break) + if self._is_punctuation_only(word): + self._last_word += word + self._emit_pending_word() + return + + # Whitespace: emit before boundary, start new segment + if word.strip() != word: + self._emit_pending_word() + self._last_word = word + self._last_timestamp = timestamp + return + + # Default: continue merging CJK characters + self._last_word += word + + def _handle_non_cjk_word_boundary(self, word: str, timestamp: float): + """Handle word boundaries for non-CJK languages. + + Non-CJK languages use spaces between words, so we emit each word separately + after merging any trailing punctuation. + + Args: + word: The word from Azure. + timestamp: Timestamp in seconds. + """ + # Punctuation: merge with previous word (don't emit yet) + if self._is_punctuation_only(word) and self._last_word is not None: + self._last_word += word + return + + # Regular word: emit previous, store current + if self._last_word is not None: + self._word_boundary_queue.put_nowait((self._last_word, self._last_timestamp)) + self._last_word = word + self._last_timestamp = timestamp async def _word_processor_task_handler(self): """Process word timestamps from the queue and call add_word_timestamps.""" @@ -397,6 +486,12 @@ def _handle_completed(self, evt): Args: evt: Completion event from Azure Speech SDK. """ + # Flush any pending word before completing + if self._last_word is not None: + self._word_boundary_queue.put_nowait((self._last_word, self._last_timestamp)) + self._last_word = None + self._last_timestamp = None + # Update cumulative audio offset for next sentence if evt.result and evt.result.audio_duration: self._cumulative_audio_offset += evt.result.audio_duration.total_seconds() @@ -435,6 +530,8 @@ def _reset_state(self): self._started = False self._first_chunk = True self._cumulative_audio_offset = 0.0 + self._last_word = None + self._last_timestamp = None async def flush_audio(self): """Flush any pending audio data.""" diff --git a/src/pipecat/services/camb/__init__.py b/src/pipecat/services/camb/__init__.py new file mode 100644 index 0000000000..f28a0edd6c --- /dev/null +++ b/src/pipecat/services/camb/__init__.py @@ -0,0 +1,5 @@ +# +# Copyright (c) 2024–2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# diff --git a/src/pipecat/services/camb/tts.py b/src/pipecat/services/camb/tts.py new file mode 100644 index 0000000000..00b4eaf9be --- /dev/null +++ b/src/pipecat/services/camb/tts.py @@ -0,0 +1,330 @@ +# +# Copyright (c) 2024–2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Camb.ai MARS text-to-speech service implementation. + +This module provides TTS functionality using Camb.ai's MARS model family, +offering high-quality text-to-speech synthesis with streaming support. + +Features: + - MARS models: mars-flash (fast), mars-pro (high quality) + - 140+ languages supported + - Real-time streaming via official SDK + - Model-specific sample rates: mars-pro (48kHz), mars-flash (22.05kHz) +""" + +from typing import Any, AsyncGenerator, Dict, Optional + +from camb import StreamTtsOutputConfiguration +from camb.client import AsyncCambAI +from loguru import logger +from pydantic import BaseModel, Field + +from pipecat.frames.frames import ( + ErrorFrame, + Frame, + StartFrame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.services.tts_service import TTSService +from pipecat.transcriptions.language import Language, resolve_language +from pipecat.utils.tracing.service_decorators import traced_tts + +# Model-specific sample rates +MODEL_SAMPLE_RATES: Dict[str, int] = { + "mars-flash": 22050, # 22.05kHz + "mars-pro": 48000, # 48kHz + "mars-instruct": 22050, # 22.05kHz +} + + +def language_to_camb_language(language: Language) -> Optional[str]: + """Convert a Pipecat Language enum to Camb.ai language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Camb.ai language code (BCP-47 format), or None if not supported. + """ + LANGUAGE_MAP = { + Language.EN: "en-us", + Language.EN_US: "en-us", + Language.EN_GB: "en-gb", + Language.EN_AU: "en-au", + Language.ES: "es-es", + Language.ES_ES: "es-es", + Language.ES_MX: "es-mx", + Language.FR: "fr-fr", + Language.FR_FR: "fr-fr", + Language.FR_CA: "fr-ca", + Language.DE: "de-de", + Language.DE_DE: "de-de", + Language.IT: "it-it", + Language.PT: "pt-pt", + Language.PT_BR: "pt-br", + Language.PT_PT: "pt-pt", + Language.NL: "nl-nl", + Language.PL: "pl-pl", + Language.RU: "ru-ru", + Language.JA: "ja-jp", + Language.KO: "ko-kr", + Language.ZH: "zh-cn", + Language.ZH_CN: "zh-cn", + Language.ZH_TW: "zh-tw", + Language.AR: "ar-sa", + Language.HI: "hi-in", + Language.TR: "tr-tr", + Language.VI: "vi-vn", + Language.TH: "th-th", + Language.ID: "id-id", + Language.MS: "ms-my", + Language.SV: "sv-se", + Language.DA: "da-dk", + Language.NO: "no-no", + Language.FI: "fi-fi", + Language.CS: "cs-cz", + Language.EL: "el-gr", + Language.HE: "he-il", + Language.HU: "hu-hu", + Language.RO: "ro-ro", + Language.SK: "sk-sk", + Language.UK: "uk-ua", + Language.BG: "bg-bg", + Language.HR: "hr-hr", + Language.SR: "sr-rs", + Language.SL: "sl-si", + Language.CA: "ca-es", + Language.EU: "eu-es", + Language.GL: "gl-es", + Language.AF: "af-za", + Language.SW: "sw-ke", + Language.TA: "ta-in", + Language.TE: "te-in", + Language.BN: "bn-in", + Language.MR: "mr-in", + Language.GU: "gu-in", + Language.KN: "kn-in", + Language.ML: "ml-in", + Language.PA: "pa-in", + Language.UR: "ur-pk", + Language.FA: "fa-ir", + Language.TL: "tl-ph", + } + + return resolve_language(language, LANGUAGE_MAP, use_base_code=True) + + +def _get_aligned_audio(buffer: bytes) -> tuple[bytes, bytes]: + """Split buffer into aligned audio (2-byte samples) and remainder. + + Args: + buffer: Raw audio bytes to align. + + Returns: + Tuple of (aligned audio bytes, remaining bytes). + """ + aligned_size = (len(buffer) // 2) * 2 + return buffer[:aligned_size], buffer[aligned_size:] + + +class CambTTSService(TTSService): + """Camb.ai MARS text-to-speech service using the official SDK. + + Converts text to speech using Camb.ai's MARS TTS models with support for + multiple languages. + + Models: + - mars-flash: Fast inference, 22.05kHz output (default) + - mars-pro: High quality, 48kHz output + + Example:: + + # Basic usage with mars-flash (fast) + tts = CambTTSService(api_key="your-api-key", model="mars-flash") + + # High quality with mars-pro + tts = CambTTSService( + api_key="your-api-key", + voice_id=12345, + model="mars-pro", + ) + """ + + class InputParams(BaseModel): + """Input parameters for Camb.ai TTS configuration. + + Parameters: + language: Language for synthesis (BCP-47 format). Defaults to English. + user_instructions: Custom instructions for mars-instruct model only. + Ignored for other models. Max 1000 characters. + """ + + language: Optional[Language] = Language.EN + user_instructions: Optional[str] = Field( + default=None, + max_length=1000, + description="Custom instructions for mars-instruct model only. " + "Use to control tone, style, or pronunciation. Max 1000 characters.", + ) + + def __init__( + self, + *, + api_key: str, + voice_id: int = 147320, + model: str = "mars-flash", + timeout: float = 60.0, + sample_rate: Optional[int] = None, + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the Camb.ai TTS service. + + Args: + api_key: Camb.ai API key for authentication. + voice_id: Voice ID to use. Defaults to 147320. + model: TTS model to use. Options: "mars-flash" (fast), "mars-pro" (high quality). + Defaults to "mars-flash". + timeout: Request timeout in seconds. Defaults to 60.0 (minimum recommended + by Camb.ai). + sample_rate: Audio sample rate in Hz. If None, uses model-specific default. + params: Additional voice parameters. If None, uses defaults. + **kwargs: Additional arguments passed to parent TTSService. + """ + super().__init__(sample_rate=sample_rate, **kwargs) + + self._api_key = api_key + self._timeout = timeout + + params = params or CambTTSService.InputParams() + + # Warn if sample rate doesn't match model's supported rate + if sample_rate and sample_rate != MODEL_SAMPLE_RATES.get(model): + logger.warning( + f"Camb.ai's {model} model only supports {MODEL_SAMPLE_RATES.get(model)}Hz " + f"sample rate. Current rate of {sample_rate}Hz may cause issues." + ) + + # Build settings + self._settings = { + "language": ( + self.language_to_service_language(params.language) if params.language else "en-us" + ), + "user_instructions": params.user_instructions, + } + + self.set_model_name(model) + self.set_voice(str(voice_id)) + self._voice_id = voice_id + + self._client = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Camb.ai service supports metrics generation. + """ + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Camb.ai language format. + + Args: + language: The language to convert. + + Returns: + The Camb.ai-specific language code, or None if not supported. + """ + return language_to_camb_language(language) + + async def start(self, frame: StartFrame): + """Start the Camb.ai TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + + self._client = AsyncCambAI(api_key=self._api_key, timeout=self._timeout) + + # Use model-specific sample rate if not explicitly specified + if not self._init_sample_rate: + self._sample_rate = MODEL_SAMPLE_RATES.get(self.model_name, 22050) + + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Camb.ai's TTS API. + + Args: + text: The text to synthesize into speech (max 3000 characters). + + Yields: + Frame: Audio frames containing the synthesized speech. + """ + logger.debug(f"{self}: Generating TTS [{text}]") + + # Validate text length + if len(text) > 3000: + logger.warning("Text too long for Camb.ai TTS (max 3000 chars), truncating") + text = text[:3000] + + try: + await self.start_ttfb_metrics() + + # Build SDK parameters + tts_kwargs: Dict[str, Any] = { + "text": text, + "voice_id": self._voice_id, + "language": self._settings["language"], + "speech_model": self.model_name, + "output_configuration": StreamTtsOutputConfiguration(format="pcm_s16le"), + } + + # Add user instructions if using mars-instruct model + if self._model_name == "mars-instruct" and self._settings.get("user_instructions"): + tts_kwargs["user_instructions"] = self._settings["user_instructions"] + + await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() + + assert self._client is not None, "Camb.ai TTS service not initialized" + + # Buffer for aligning chunks to 2-byte boundaries (16-bit PCM) + audio_buffer = b"" + + # Stream audio chunks from SDK + async for chunk in self._client.text_to_speech.tts(**tts_kwargs): + if chunk: + await self.stop_ttfb_metrics() + audio_buffer += chunk + + # Only yield complete 16-bit samples (2 bytes per sample) + aligned_audio, audio_buffer = _get_aligned_audio(audio_buffer) + if aligned_audio: + yield TTSAudioRawFrame( + audio=aligned_audio, + sample_rate=self.sample_rate, + num_channels=1, + ) + + # Yield any remaining complete samples + if len(audio_buffer) >= 2: + aligned_audio, _ = _get_aligned_audio(audio_buffer) + if aligned_audio: + yield TTSAudioRawFrame( + audio=aligned_audio, + sample_rate=self.sample_rate, + num_channels=1, + ) + + except Exception as e: + yield ErrorFrame(error=f"Camb.ai TTS error: {e}") + finally: + yield TTSStoppedFrame() diff --git a/src/pipecat/services/cerebras/llm.py b/src/pipecat/services/cerebras/llm.py index 9ceb489057..54ea45ddbe 100644 --- a/src/pipecat/services/cerebras/llm.py +++ b/src/pipecat/services/cerebras/llm.py @@ -6,8 +6,6 @@ """Cerebras LLM service implementation using OpenAI-compatible interface.""" -from typing import List - from loguru import logger from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index 13b72bcf7f..045af9811c 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -27,7 +27,6 @@ UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) -from pipecat.processors.frame_processor import FrameDirection from pipecat.services.stt_service import WebsocketSTTService from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 @@ -676,8 +675,7 @@ async def _handle_end_of_turn(self, transcript: str, data: Dict[str, Any]): await self._handle_transcription(transcript, True, self._language) await self.stop_processing_metrics() - await self.push_frame(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM) - await self.push_frame(UserStoppedSpeakingFrame(), FrameDirection.UPSTREAM) + await self.broadcast_frame(UserStoppedSpeakingFrame) await self._call_event_handler("on_end_of_turn", transcript) async def _handle_eager_end_of_turn(self, transcript: str, data: Dict[str, Any]): diff --git a/src/pipecat/services/deepseek/llm.py b/src/pipecat/services/deepseek/llm.py index 50bdebd3be..56f1ddd189 100644 --- a/src/pipecat/services/deepseek/llm.py +++ b/src/pipecat/services/deepseek/llm.py @@ -6,8 +6,6 @@ """DeepSeek LLM service implementation using OpenAI-compatible interface.""" -from typing import List - from loguru import logger from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams diff --git a/src/pipecat/services/fireworks/llm.py b/src/pipecat/services/fireworks/llm.py index 92467786f4..d7bf579081 100644 --- a/src/pipecat/services/fireworks/llm.py +++ b/src/pipecat/services/fireworks/llm.py @@ -6,8 +6,6 @@ """Fireworks AI service implementation using OpenAI-compatible interface.""" -from typing import List - from loguru import logger from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams diff --git a/src/pipecat/services/gemini_multimodal_live/__init__.py b/src/pipecat/services/gemini_multimodal_live/__init__.py index 513d9fd66f..ac4524606c 100644 --- a/src/pipecat/services/gemini_multimodal_live/__init__.py +++ b/src/pipecat/services/gemini_multimodal_live/__init__.py @@ -1,2 +1,7 @@ from .file_api import GeminiFileAPI from .gemini import GeminiMultimodalLiveLLMService + +__all__ = [ + "GeminiFileAPI", + "GeminiMultimodalLiveLLMService", +] diff --git a/src/pipecat/services/google/gemini_live/__init__.py b/src/pipecat/services/google/gemini_live/__init__.py index 142ca2a833..f4bfbb5c84 100644 --- a/src/pipecat/services/google/gemini_live/__init__.py +++ b/src/pipecat/services/google/gemini_live/__init__.py @@ -1,3 +1,9 @@ from .file_api import GeminiFileAPI from .llm import GeminiLiveLLMService from .llm_vertex import GeminiLiveVertexLLMService + +__all__ = [ + "GeminiFileAPI", + "GeminiLiveLLMService", + "GeminiLiveVertexLLMService", +] diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 13b5fb18cc..f61c9826c8 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1710,11 +1710,26 @@ async def _handle_msg_output_transcription(self, message: LiveServerMessage): await self.push_frame(TTSStartedFrame()) await self.push_frame(LLMFullResponseStartFrame()) - frame = TTSTextFrame(text=text, aggregated_by=AggregationType.SENTENCE) - # Gemini Live text already includes any necessary inter-chunk spaces - frame.includes_inter_frame_spaces = True - - await self.push_frame(frame) + await self._push_output_transcription_text_frames(text) + + async def _push_output_transcription_text_frames(self, text: str): + # In a typical "cascade" LLM + TTS setup, LLMTextFrames would not + # proceed beyond the TTS service. Therefore, since a speech-to-speech + # service like Gemini Live combines both LLM and TTS functionality, you + # might think we wouldn't need to push LLMTextFrames at all. However, + # RTVI relies on LLMTextFrames being pushed to trigger its + # "bot-llm-text" event. So here we push an LLMTextFrame, too, but avoid + # appending it to context to avoid context message duplication. + + # Push LLMTextFrame + llm_text_frame = LLMTextFrame(text) + llm_text_frame.append_to_context = False + await self.push_frame(llm_text_frame) + + # Push TTSTextFrame + tts_text_frame = TTSTextFrame(text, aggregated_by=AggregationType.SENTENCE) + tts_text_frame.includes_inter_frame_spaces = True + await self.push_frame(tts_text_frame) async def _handle_msg_grounding_metadata(self, message: LiveServerMessage): """Handle dedicated grounding metadata messages.""" diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 20341b64d7..adb2664bba 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -40,7 +40,6 @@ LLMThoughtStartFrame, LLMThoughtTextFrame, LLMUpdateSettingsFrame, - OutputImageRawFrame, UserImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage diff --git a/src/pipecat/services/google/rtvi.py b/src/pipecat/services/google/rtvi.py index 1cc68f5e49..1ef70d67da 100644 --- a/src/pipecat/services/google/rtvi.py +++ b/src/pipecat/services/google/rtvi.py @@ -15,9 +15,7 @@ from pydantic import BaseModel -from pipecat.frames.frames import Frame from pipecat.observers.base_observer import FramePushed -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index ac77f04505..dcf0adf89b 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -29,7 +29,6 @@ from pipecat.frames.frames import ( CancelFrame, EndFrame, - ErrorFrame, Frame, InterimTranscriptionFrame, StartFrame, diff --git a/src/pipecat/services/gradium/tts.py b/src/pipecat/services/gradium/tts.py index 14e0935411..df33753ced 100644 --- a/src/pipecat/services/gradium/tts.py +++ b/src/pipecat/services/gradium/tts.py @@ -6,7 +6,6 @@ import base64 import json -import uuid from typing import Any, AsyncGenerator, Mapping, Optional from loguru import logger diff --git a/src/pipecat/services/grok/realtime/llm.py b/src/pipecat/services/grok/realtime/llm.py index cbf9b4d5c3..e1355ce313 100644 --- a/src/pipecat/services/grok/realtime/llm.py +++ b/src/pipecat/services/grok/realtime/llm.py @@ -33,6 +33,7 @@ LLMFullResponseStartFrame, LLMMessagesAppendFrame, LLMSetToolsFrame, + LLMTextFrame, LLMUpdateSettingsFrame, StartFrame, TranscriptionFrame, @@ -619,9 +620,26 @@ async def _handle_evt_response_done(self, evt): async def _handle_evt_audio_transcript_delta(self, evt): """Handle audio transcript delta event.""" if evt.delta: - frame = TTSTextFrame(evt.delta, aggregated_by=AggregationType.SENTENCE) - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self._push_output_transcript_text_frames(evt.delta) + + async def _push_output_transcript_text_frames(self, text: str): + # In a typical "cascade" LLM + TTS setup, LLMTextFrames would not + # proceed beyond the TTS service. Therefore, since a speech-to-speech + # service like Grok Realtime combines both LLM and TTS functionality, + # you might think we wouldn't need to push LLMTextFrames at all. + # However, RTVI relies on LLMTextFrames being pushed to trigger its + # "bot-llm-text" event. So here we push an LLMTextFrame, too, but avoid + # appending it to context to avoid context message duplication. + + # Push LLMTextFrame + llm_text_frame = LLMTextFrame(text) + llm_text_frame.append_to_context = False + await self.push_frame(llm_text_frame) + + # Push TTSTextFrame + tts_text_frame = TTSTextFrame(text, aggregated_by=AggregationType.SENTENCE) + tts_text_frame.includes_inter_frame_spaces = True + await self.push_frame(tts_text_frame) async def _handle_evt_function_call_arguments_done(self, evt): """Handle function call arguments done event.""" @@ -659,7 +677,7 @@ async def _handle_evt_speech_stopped(self, evt): """Handle speech stopped event from VAD.""" await self.start_ttfb_metrics() await self.start_processing_metrics() - await self.push_frame(UserStoppedSpeakingFrame()) + await self.broadcast_frame(UserStoppedSpeakingFrame) async def _handle_evt_error(self, evt): """Handle error event.""" @@ -734,6 +752,14 @@ async def _process_completed_function_calls(self, send_new_results: bool): async def _send_user_audio(self, frame): """Send user audio to Grok.""" + # Don't send audio if conversation setup is still pending, as it can + # lead to errors. For example: audio sent before conversation setup + # will be interpreted as having Grok's default sample rate (24000), + # and if that differs from the sample rate we eventually set through + # the conversation setup, Grok will error out. + if self._llm_needs_conversation_setup: + return + payload = base64.b64encode(frame.audio).decode("utf-8") await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload)) diff --git a/src/pipecat/services/hathora/__init__.py b/src/pipecat/services/hathora/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/pipecat/services/hathora/stt.py b/src/pipecat/services/hathora/stt.py new file mode 100644 index 0000000000..8aadb6b655 --- /dev/null +++ b/src/pipecat/services/hathora/stt.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""[Hathora-hosted](https://models.hathora.dev) speech-to-text services.""" + +import base64 +import os +from typing import AsyncGenerator, Optional + +import aiohttp +from pydantic import BaseModel + +from pipecat.frames.frames import ( + ErrorFrame, + Frame, + TranscriptionFrame, +) +from pipecat.services.stt_service import SegmentedSTTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.tracing.service_decorators import traced_stt + +from .utils import ConfigOption + + +class HathoraSTTService(SegmentedSTTService): + """This service supports several different speech-to-text models hosted by Hathora. + + [Documentation](https://models.hathora.dev) + """ + + class InputParams(BaseModel): + """Optional input parameters for Hathora STT configuration. + + Parameters: + language: Language code (if supported by model). + config: Some models support additional config, refer to + [docs](https://models.hathora.dev) for each model to see + what is supported. + """ + + language: Optional[str] = None + config: Optional[list[ConfigOption]] = None + + def __init__( + self, + *, + model: str, + sample_rate: Optional[int] = None, + api_key: Optional[str] = None, + base_url: str = "https://api.models.hathora.dev/inference/v1/stt", + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the Hathora STT service. + + Args: + model: Model to use; find available models + [here](https://models.hathora.dev). + sample_rate: The sample rate for audio input. If None, will be determined + from the start frame. + api_key: API key for authentication with the Hathora service; + provision one [here](https://models.hathora.dev/tokens). + base_url: Base API URL for the Hathora STT service. + params: Configuration parameters. + **kwargs: Additional arguments passed to the parent class. + """ + super().__init__( + sample_rate=sample_rate, + **kwargs, + ) + self._model = model + self._api_key = api_key or os.getenv("HATHORA_API_KEY") + self._base_url = base_url + + params = params or HathoraSTTService.InputParams() + + self._settings = { + "language": params.language, + "config": params.config, + } + + self.set_model_name(model) + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True + """ + return True + + @traced_stt + async def _handle_transcription( + self, transcript: str, is_final: bool, language: Optional[Language] = None + ): + """Handle a transcription result with tracing.""" + pass + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Run speech-to-text on the provided audio data. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + Frame: Frames containing transcription results (typically TextFrame). + """ + try: + await self.start_processing_metrics() + await self.start_ttfb_metrics() + + url = f"{self._base_url}" + + payload = { + "model": self._model, + } + + if self._settings["language"] is not None: + payload["language"] = self._settings["language"] + if self._settings["config"] is not None: + payload["model_config"] = [ + {"name": option.name, "value": option.value} + for option in self._settings["config"] + ] + + base64_audio = base64.b64encode(audio).decode("utf-8") + payload["audio"] = base64_audio + + async with aiohttp.ClientSession() as session: + async with session.post( + url, + headers={"Authorization": f"Bearer {self._api_key}"}, + json=payload, + ) as resp: + response = await resp.json() + + if response and "text" in response: + text = response["text"].strip() + if text: # Only yield non-empty text + # Hathora's API currently doesn't return language info + # so we default to the requested language or "en" + response_language = self._settings["language"] or "en" + await self._handle_transcription(text, True, response_language) + yield TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + Language(response_language), + result=response, + ) + + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") diff --git a/src/pipecat/services/hathora/tts.py b/src/pipecat/services/hathora/tts.py new file mode 100644 index 0000000000..e59c4ad462 --- /dev/null +++ b/src/pipecat/services/hathora/tts.py @@ -0,0 +1,173 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""[Hathora-hosted](https://models.hathora.dev) text-to-speech services.""" + +import io +import os +import wave +from typing import AsyncGenerator, Optional, Tuple + +import aiohttp +from pydantic import BaseModel + +from pipecat.frames.frames import ( + ErrorFrame, + Frame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.services.tts_service import TTSService +from pipecat.utils.tracing.service_decorators import traced_tts + +from .utils import ConfigOption + + +def _decode_audio_payload( + audio_bytes: bytes, + *, + fallback_sample_rate: int = 24000, + fallback_channels: int = 1, +) -> Tuple[bytes, int, int]: + """Convert a WAV/PCM payload into raw PCM samples for TTSAudioRawFrame.""" + try: + with wave.open(io.BytesIO(audio_bytes), "rb") as wav_reader: + channels = wav_reader.getnchannels() + sample_rate = wav_reader.getframerate() + frames = wav_reader.readframes(wav_reader.getnframes()) + return frames, sample_rate, channels + except (wave.Error, EOFError): + # If the payload is already raw PCM, just pass it through. + return audio_bytes, fallback_sample_rate, fallback_channels + + +class HathoraTTSService(TTSService): + """This service supports several different text-to-speech models hosted by Hathora. + + [Documentation](https://models.hathora.dev) + """ + + class InputParams(BaseModel): + """Optional input parameters for Hathora TTS configuration. + + Parameters: + speed: Speech speed multiplier (if supported by model). + config: Some models support additional config, refer to + [docs](https://models.hathora.dev) for each model to see + what is supported. + """ + + speed: Optional[float] = None + config: Optional[list[ConfigOption]] = None + + def __init__( + self, + *, + model: str, + voice_id: Optional[str] = None, + sample_rate: Optional[int] = None, + api_key: Optional[str] = None, + base_url: str = "https://api.models.hathora.dev/inference/v1/tts", + params: Optional[InputParams] = None, + **kwargs, + ): + """Initialize the Hathora TTS service. + + Args: + model: Model to use; find available models + [here](https://models.hathora.dev). + voice_id: Voice to use for synthesis (if supported by model). + sample_rate: Output sample rate for generated audio. + api_key: API key for authentication with the Hathora service; + provision one [here](https://models.hathora.dev/tokens). + base_url: Base API URL for the Hathora TTS service. + params: Configuration parameters. + **kwargs: Additional arguments passed to the parent class. + """ + super().__init__( + sample_rate=sample_rate, + **kwargs, + ) + self._model = model + self._api_key = api_key or os.getenv("HATHORA_API_KEY") + self._base_url = base_url + + params = params or HathoraTTSService.InputParams() + + self._settings = { + "speed": params.speed, + "config": params.config, + } + + self.set_model_name(model) + self.set_voice(voice_id) + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True + """ + return True + + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Run text-to-speech synthesis on the provided text. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ + try: + await self.start_processing_metrics() + await self.start_ttfb_metrics() + + url = f"{self._base_url}" + + payload = {"model": self._model, "text": text} + + if self._voice_id is not None: + payload["voice"] = self._voice_id + if self._settings["speed"] is not None: + payload["speed"] = self._settings["speed"] + if self._settings["config"] is not None: + payload["model_config"] = [ + {"name": option.name, "value": option.value} + for option in self._settings["config"] + ] + + yield TTSStartedFrame() + + async with aiohttp.ClientSession() as session: + async with session.post( + url, + headers={"Authorization": f"Bearer {self._api_key}"}, + json=payload, + ) as resp: + audio_data = await resp.read() + + pcm_audio, sample_rate, num_channels = _decode_audio_payload( + audio_data, + fallback_sample_rate=self.sample_rate, + ) + + frame = TTSAudioRawFrame( + audio=pcm_audio, + sample_rate=self.sample_rate, + num_channels=num_channels, + ) + + yield frame + + except Exception as e: + yield ErrorFrame(error=f"Unknown error occurred: {e}") + finally: + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + yield TTSStoppedFrame() diff --git a/src/pipecat/services/hathora/utils.py b/src/pipecat/services/hathora/utils.py new file mode 100644 index 0000000000..6d8eb54b89 --- /dev/null +++ b/src/pipecat/services/hathora/utils.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Utilities and types for [Hathora-hosted](https://models.hathora.dev) voice services.""" + +from dataclasses import dataclass + + +@dataclass +class ConfigOption: + """Extra configuration option passed into model_config for Hathora (if supported by model). + + Args: + name: Name of the configuration option. + value: Value of the configuration option. + """ + + name: str + value: str diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index c169cf4c0a..7f89438163 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -16,7 +16,6 @@ from pipecat.frames.frames import ( CancelFrame, EndFrame, - ErrorFrame, Frame, InterruptionFrame, StartFrame, diff --git a/src/pipecat/services/mem0/memory.py b/src/pipecat/services/mem0/memory.py index afb38088ea..c4dfe16ded 100644 --- a/src/pipecat/services/mem0/memory.py +++ b/src/pipecat/services/mem0/memory.py @@ -16,7 +16,7 @@ from loguru import logger from pydantic import BaseModel, Field -from pipecat.frames.frames import ErrorFrame, Frame, LLMContextFrame, LLMMessagesFrame +from pipecat.frames.frames import Frame, LLMContextFrame, LLMMessagesFrame from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, @@ -121,7 +121,6 @@ def _store_messages(self, messages: List[Dict[str, Any]]): try: logger.debug(f"Storing {len(messages)} messages in Mem0") params = { - "async_mode": True, "messages": messages, "metadata": {"platform": "pipecat"}, "output_format": "v1.1", diff --git a/src/pipecat/services/mistral/llm.py b/src/pipecat/services/mistral/llm.py index b95c3d1ab0..54361ef28d 100644 --- a/src/pipecat/services/mistral/llm.py +++ b/src/pipecat/services/mistral/llm.py @@ -9,12 +9,10 @@ from typing import List, Sequence from loguru import logger -from openai import AsyncStream -from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam +from openai.types.chat import ChatCompletionMessageParam from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams from pipecat.frames.frames import FunctionCallFromLLM -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai.llm import OpenAILLMService diff --git a/src/pipecat/services/moondream/vision.py b/src/pipecat/services/moondream/vision.py index f0ae8dca2d..6a180b4cb4 100644 --- a/src/pipecat/services/moondream/vision.py +++ b/src/pipecat/services/moondream/vision.py @@ -46,7 +46,7 @@ def detect_device(): and dtype is the recommended torch data type for that device. """ try: - import intel_extension_for_pytorch + import intel_extension_for_pytorch # noqa: F401 if torch.xpu.is_available(): return torch.device("xpu"), torch.float32 diff --git a/src/pipecat/services/nvidia/stt.py b/src/pipecat/services/nvidia/stt.py index 0d671571f7..7d52f51307 100644 --- a/src/pipecat/services/nvidia/stt.py +++ b/src/pipecat/services/nvidia/stt.py @@ -134,6 +134,7 @@ def __init__( params = params or NvidiaSTTService.InputParams() + self._server = server self._api_key = api_key self._use_ssl = use_ssl self._profanity_filter = False @@ -162,18 +163,53 @@ def __init__( self.set_model_name(model_function_map.get("model_name")) + self._asr_service = None + self._queue = None + self._config = None + self._thread_task = None + + def _initialize_client(self): metadata = [ ["function-id", self._function_id], - ["authorization", f"Bearer {api_key}"], + ["authorization", f"Bearer {self._api_key}"], ] - auth = riva.client.Auth(None, self._use_ssl, server, metadata) + auth = riva.client.Auth(None, self._use_ssl, self._server, metadata) self._asr_service = riva.client.ASRService(auth) - self._queue = None - self._config = None - self._thread_task = None - self._response_task = None + def _create_recognition_config(self): + """Create the NVIDIA Riva ASR recognition configuration.""" + config = riva.client.StreamingRecognitionConfig( + config=riva.client.RecognitionConfig( + encoding=riva.client.AudioEncoding.LINEAR_PCM, + language_code=self._language_code, + model="", + max_alternatives=1, + profanity_filter=self._profanity_filter, + enable_automatic_punctuation=self._automatic_punctuation, + verbatim_transcripts=not self._no_verbatim_transcripts, + sample_rate_hertz=self.sample_rate, + audio_channel_count=1, + ), + interim_results=True, + ) + + riva.client.add_word_boosting_to_config( + config, self._boosted_lm_words, self._boosted_lm_score + ) + + riva.client.add_endpoint_parameters_to_config( + config, + self._start_history, + self._start_threshold, + self._stop_history, + self._stop_history_eou, + self._stop_threshold, + self._stop_threshold_eou, + ) + riva.client.add_custom_configuration_to_config(config, self._custom_configuration) + + return config def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -206,49 +242,15 @@ async def start(self, frame: StartFrame): frame: StartFrame indicating pipeline start. """ await super().start(frame) + self._initialize_client() + self._config = self._create_recognition_config() - if self._config: - return - - config = riva.client.StreamingRecognitionConfig( - config=riva.client.RecognitionConfig( - encoding=riva.client.AudioEncoding.LINEAR_PCM, - language_code=self._language_code, - model="", - max_alternatives=1, - profanity_filter=self._profanity_filter, - enable_automatic_punctuation=self._automatic_punctuation, - verbatim_transcripts=not self._no_verbatim_transcripts, - sample_rate_hertz=self.sample_rate, - audio_channel_count=1, - ), - interim_results=True, - ) - - riva.client.add_word_boosting_to_config( - config, self._boosted_lm_words, self._boosted_lm_score - ) - - riva.client.add_endpoint_parameters_to_config( - config, - self._start_history, - self._start_threshold, - self._stop_history, - self._stop_history_eou, - self._stop_threshold, - self._stop_threshold_eou, - ) - riva.client.add_custom_configuration_to_config(config, self._custom_configuration) - - self._config = config self._queue = asyncio.Queue() if not self._thread_task: self._thread_task = self.create_task(self._thread_task_handler()) - if not self._response_task: - self._response_queue = asyncio.Queue() - self._response_task = self.create_task(self._response_task_handler()) + logger.debug(f"Initialized NvidiaSTTService with model: {self.model_name}") async def stop(self, frame: EndFrame): """Stop the NVIDIA Riva STT service and clean up resources. @@ -273,10 +275,6 @@ async def _stop_tasks(self): await self.cancel_task(self._thread_task) self._thread_task = None - if self._response_task: - await self.cancel_task(self._response_task) - self._response_task = None - def _response_handler(self): responses = self._asr_service.streaming_response_generator( audio_chunks=self, @@ -285,9 +283,7 @@ def _response_handler(self): for response in responses: if not response.results: continue - asyncio.run_coroutine_threadsafe( - self._response_queue.put(response), self.get_event_loop() - ) + asyncio.run_coroutine_threadsafe(self._handle_response(response), self.get_event_loop()) async def _thread_task_handler(self): try: @@ -339,12 +335,6 @@ async def _handle_response(self, response): ) ) - async def _response_task_handler(self): - while True: - response = await self._response_queue.get() - await self._handle_response(response) - self._response_queue.task_done() - async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: """Process audio data for speech-to-text transcription. @@ -503,8 +493,6 @@ def _initialize_client(self): auth = riva.client.Auth(None, self._use_ssl, self._server, metadata) self._asr_service = riva.client.ASRService(auth) - logger.info(f"Initialized NvidiaSegmentedSTTService with model: {self.model_name}") - def _create_recognition_config(self): """Create the NVIDIA Riva ASR recognition configuration.""" # Create base configuration @@ -572,6 +560,7 @@ async def start(self, frame: StartFrame): await super().start(frame) self._initialize_client() self._config = self._create_recognition_config() + logger.debug(f"Initialized NvidiaSegmentedSTTService with model: {self.model_name}") async def set_language(self, language: Language): """Set the language for the STT service. @@ -605,21 +594,12 @@ async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: Frame: TranscriptionFrame containing the transcribed text. """ try: - await self.start_processing_metrics() - await self.start_ttfb_metrics() - - # Make sure the client is initialized - if self._asr_service is None: - self._initialize_client() - - # Make sure the config is created - if self._config is None: - self._config = self._create_recognition_config() - - # Type assertion to satisfy the IDE assert self._asr_service is not None, "ASR service not initialized" assert self._config is not None, "Recognition config not created" + await self.start_processing_metrics() + await self.start_ttfb_metrics() + # Process audio with NVIDIA Riva ASR - explicitly request non-future response raw_response = self._asr_service.offline_recognize(audio, self._config, future=False) @@ -627,43 +607,40 @@ async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: await self.stop_processing_metrics() # Process the response - handle different possible return types - try: - # If it's a future-like object, get the result - if hasattr(raw_response, "result"): - response = raw_response.result() - else: - response = raw_response - - # Process transcription results - transcription_found = False - - # Now we can safely check results - # Type hint for the IDE - results = getattr(response, "results", []) - - for result in results: - alternatives = getattr(result, "alternatives", []) - if alternatives: - text = alternatives[0].transcript.strip() - if text: - logger.debug(f"Transcription: [{text}]") - yield TranscriptionFrame( - text, - self._user_id, - time_now_iso8601(), - self._language_enum, - ) - transcription_found = True - - await self._handle_transcription(text, True, self._language_enum) - - if not transcription_found: - logger.debug("No transcription results found in NVIDIA Riva response") - - except AttributeError as ae: - logger.error(f"Unexpected response structure from NVIDIA Riva: {ae}") - yield ErrorFrame(f"Unexpected NVIDIA Riva response format: {str(ae)}") + # If it's a future-like object, get the result + if hasattr(raw_response, "result"): + response = raw_response.result() + else: + response = raw_response + + # Process transcription results + transcription_found = False + + # Now we can safely check results + # Type hint for the IDE + results = getattr(response, "results", []) + + for result in results: + alternatives = getattr(result, "alternatives", []) + if alternatives: + text = alternatives[0].transcript.strip() + if text: + logger.debug(f"Transcription: [{text}]") + yield TranscriptionFrame( + text, + self._user_id, + time_now_iso8601(), + self._language_enum, + ) + transcription_found = True + + await self._handle_transcription(text, True, self._language_enum) + if not transcription_found: + logger.debug(f"{self}: No transcription results found in NVIDIA Riva response") + except AttributeError as ae: + logger.error(f"{self}: Unexpected response structure from NVIDIA Riva: {ae}") + yield ErrorFrame(f"{self}: Unexpected NVIDIA Riva response format: {str(ae)}") except Exception as e: logger.error(f"{self} exception: {e}") yield ErrorFrame(error=f"{self} error: {e}") diff --git a/src/pipecat/services/nvidia/tts.py b/src/pipecat/services/nvidia/tts.py index 70be24a8fd..eddafce017 100644 --- a/src/pipecat/services/nvidia/tts.py +++ b/src/pipecat/services/nvidia/tts.py @@ -12,7 +12,7 @@ import asyncio import os -from typing import AsyncGenerator, Mapping, Optional +from typing import AsyncGenerator, AsyncIterable, Generator, Mapping, Optional from pipecat.utils.tracing.service_decorators import traced_tts @@ -25,6 +25,7 @@ from pipecat.frames.frames import ( ErrorFrame, Frame, + StartFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, @@ -34,14 +35,12 @@ try: import riva.client - + import riva.client.proto.riva_tts_pb2 as rtts except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use NVIDIA Riva TTS, you need to `pip install pipecat-ai[nvidia]`.") raise Exception(f"Missing module: {e}") -NVIDIA_TTS_TIMEOUT_SECS = 5 - class NvidiaTTSService(TTSService): """NVIDIA Riva text-to-speech service. @@ -93,6 +92,7 @@ def __init__( params = params or NvidiaTTSService.InputParams() + self._server = server self._api_key = api_key self._voice_id = voice_id self._language_code = params.language @@ -102,18 +102,8 @@ def __init__( self.set_model_name(model_function_map.get("model_name")) self.set_voice(voice_id) - metadata = [ - ["function-id", self._function_id], - ["authorization", f"Bearer {api_key}"], - ] - auth = riva.client.Auth(None, self._use_ssl, server, metadata) - - self._service = riva.client.SpeechSynthesisService(auth) - - # warm up the service - config_response = self._service.stub.GetRivaSynthesisConfig( - riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest() - ) + self._service = None + self._config = None async def set_model(self, model: str): """Attempt to set the TTS model. @@ -129,6 +119,39 @@ async def set_model(self, model: str): f"{self.__class__.__name__}(api_key=, model_function_map={example})" ) + def _initialize_client(self): + if self._service is not None: + return + + metadata = [ + ["function-id", self._function_id], + ["authorization", f"Bearer {self._api_key}"], + ] + auth = riva.client.Auth(None, self._use_ssl, self._server, metadata) + + self._service = riva.client.SpeechSynthesisService(auth) + + def _create_synthesis_config(self): + if not self._service: + return + + # warm up the service + config = self._service.stub.GetRivaSynthesisConfig( + riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest() + ) + return config + + async def start(self, frame: StartFrame): + """Start the Cartesia TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + self._initialize_client() + self._config = self._create_synthesis_config() + logger.debug(f"Initialized NvidiaTTSService with model: {self.model_name}") + @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using NVIDIA Riva TTS. @@ -140,39 +163,43 @@ async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: Frame: Audio frames containing the synthesized speech data. """ - def read_audio_responses(queue: asyncio.Queue): - def add_response(r): - asyncio.run_coroutine_threadsafe(queue.put(r), self.get_event_loop()) - + def read_audio_responses() -> Generator[rtts.SynthesizeSpeechResponse, None, None]: + responses = self._service.synthesize_online( + text, + self._voice_id, + self._language_code, + sample_rate_hz=self.sample_rate, + zero_shot_audio_prompt_file=None, + zero_shot_quality=self._quality, + custom_dictionary={}, + ) + return responses + + def async_next(it): try: - responses = self._service.synthesize_online( - text, - self._voice_id, - self._language_code, - sample_rate_hz=self.sample_rate, - zero_shot_audio_prompt_file=None, - zero_shot_quality=self._quality, - custom_dictionary={}, - ) - for r in responses: - add_response(r) - add_response(None) - except Exception as e: - logger.error(f"{self} exception: {e}") - add_response(None) - - await self.start_ttfb_metrics() - yield TTSStartedFrame() + return next(it) + except StopIteration: + return None - logger.debug(f"{self}: Generating TTS [{text}]") + async def async_iterator(iterator) -> AsyncIterable[rtts.SynthesizeSpeechResponse]: + while True: + item = await asyncio.to_thread(async_next, iterator) + if item is None: + return + yield item try: - queue = asyncio.Queue() - await asyncio.to_thread(read_audio_responses, queue) + assert self._service is not None, "TTS service not initialized" + assert self._config is not None, "Synthesis configuration not created" + + await self.start_ttfb_metrics() + yield TTSStartedFrame() + + logger.debug(f"{self}: Generating TTS [{text}]") - # Wait for the thread to start. - resp = await asyncio.wait_for(queue.get(), timeout=NVIDIA_TTS_TIMEOUT_SECS) - while resp: + responses = await asyncio.to_thread(read_audio_responses) + + async for resp in async_iterator(responses): await self.stop_ttfb_metrics() frame = TTSAudioRawFrame( audio=resp.audio, @@ -180,10 +207,12 @@ def add_response(r): num_channels=1, ) yield frame - resp = await asyncio.wait_for(queue.get(), timeout=NVIDIA_TTS_TIMEOUT_SECS) + + await self.start_tts_usage_metrics(text) + yield TTSStoppedFrame() except asyncio.TimeoutError: logger.error(f"{self} timeout waiting for audio response") yield ErrorFrame(error=f"{self} error: {e}") - - await self.start_tts_usage_metrics(text) - yield TTSStoppedFrame() + except Exception as e: + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py index c0ef5b8906..0aa1355e6e 100644 --- a/src/pipecat/services/openai/realtime/events.py +++ b/src/pipecat/services/openai/realtime/events.py @@ -1002,17 +1002,14 @@ class TokenDetails(BaseModel): image_tokens: Number of image tokens used (for input only). """ + model_config = ConfigDict(extra="allow") + cached_tokens: Optional[int] = 0 text_tokens: Optional[int] = 0 audio_tokens: Optional[int] = 0 cached_tokens_details: Optional[CachedTokensDetails] = None image_tokens: Optional[int] = 0 - class Config: - """Pydantic configuration for TokenDetails.""" - - extra = "allow" - class Usage(BaseModel): """Token usage statistics for a response. diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 598d4f510c..11a83741e1 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -724,10 +724,26 @@ async def _handle_evt_audio_transcript_delta(self, evt): # We receive audio transcript deltas (as opposed to text deltas) when # the output modality is "audio" (the default) if evt.delta: - frame = TTSTextFrame(evt.delta, aggregated_by=AggregationType.SENTENCE) - # OpenAI Realtime text already includes any necessary inter-chunk spaces - frame.includes_inter_frame_spaces = True - await self.push_frame(frame) + await self._push_output_transcript_text_frames(evt.delta) + + async def _push_output_transcript_text_frames(self, text: str): + # In a typical "cascade" LLM + TTS setup, LLMTextFrames would not + # proceed beyond the TTS service. Therefore, since a speech-to-speech + # service like OpenAI Realtime combines both LLM and TTS functionality, + # you might think we wouldn't need to push LLMTextFrames at all. + # However, RTVI relies on LLMTextFrames being pushed to trigger its + # "bot-llm-text" event. So here we push an LLMTextFrame, too, but avoid + # appending it to context to avoid context message duplication. + + # Push LLMTextFrame + llm_text_frame = LLMTextFrame(text) + llm_text_frame.append_to_context = False + await self.push_frame(llm_text_frame) + + # Push TTSTextFrame + tts_text_frame = TTSTextFrame(text, aggregated_by=AggregationType.SENTENCE) + tts_text_frame.includes_inter_frame_spaces = True + await self.push_frame(tts_text_frame) async def _handle_evt_function_call_arguments_done(self, evt): """Handle completion of function call arguments. @@ -776,7 +792,7 @@ async def _handle_evt_speech_started(self, evt): async def _handle_evt_speech_stopped(self, evt): await self.start_ttfb_metrics() await self.start_processing_metrics() - await self.push_frame(UserStoppedSpeakingFrame()) + await self.broadcast_frame(UserStoppedSpeakingFrame) async def _maybe_handle_evt_retrieve_conversation_item_error(self, evt: events.ErrorEvent): """Maybe handle an error event related to retrieving a conversation item. diff --git a/src/pipecat/services/openai_realtime/__init__.py b/src/pipecat/services/openai_realtime/__init__.py index a302c783be..90729ef317 100644 --- a/src/pipecat/services/openai_realtime/__init__.py +++ b/src/pipecat/services/openai_realtime/__init__.py @@ -25,3 +25,13 @@ DeprecationWarning, stacklevel=2, ) + +__all__ = [ + "AzureRealtimeLLMService", + "InputAudioNoiseReduction", + "InputAudioTranscription", + "SemanticTurnDetection", + "SessionProperties", + "TurnDetection", + "OpenAIRealtimeLLMService", +] diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index 595105d7f6..b7c976bb60 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -7,3 +7,13 @@ TurnDetection, ) from .openai import OpenAIRealtimeBetaLLMService + +__all__ = [ + "AzureRealtimeBetaLLMService", + "InputAudioNoiseReduction", + "InputAudioTranscription", + "SemanticTurnDetection", + "SessionProperties", + "TurnDetection", + "OpenAIRealtimeBetaLLMService", +] diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index ce582e9bd9..596a11a3b3 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -877,15 +877,12 @@ class TokenDetails(BaseModel): audio_tokens: Number of audio tokens used. Defaults to 0. """ + model_config = ConfigDict(extra="allow") + cached_tokens: Optional[int] = 0 text_tokens: Optional[int] = 0 audio_tokens: Optional[int] = 0 - class Config: - """Pydantic configuration for TokenDetails.""" - - extra = "allow" - class Usage(BaseModel): """Token usage statistics for a response. diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index bea26c3e6f..128bcc93a5 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -662,7 +662,7 @@ async def _handle_evt_speech_started(self, evt): async def _handle_evt_speech_stopped(self, evt): await self.start_ttfb_metrics() await self.start_processing_metrics() - await self.push_frame(UserStoppedSpeakingFrame()) + await self.broadcast_frame(UserStoppedSpeakingFrame) async def _maybe_handle_evt_retrieve_conversation_item_error(self, evt: events.ErrorEvent): """Maybe handle an error event related to retrieving a conversation item. diff --git a/src/pipecat/services/openpipe/llm.py b/src/pipecat/services/openpipe/llm.py index f3e95f71b8..fa53c15547 100644 --- a/src/pipecat/services/openpipe/llm.py +++ b/src/pipecat/services/openpipe/llm.py @@ -10,7 +10,7 @@ enabling integration with OpenPipe's fine-tuning and monitoring capabilities. """ -from typing import Dict, List, Optional +from typing import Dict, Optional from loguru import logger diff --git a/src/pipecat/services/ultravox/llm.py b/src/pipecat/services/ultravox/llm.py index 489ba367da..d549b11e58 100644 --- a/src/pipecat/services/ultravox/llm.py +++ b/src/pipecat/services/ultravox/llm.py @@ -19,7 +19,6 @@ import aiohttp from loguru import logger -from openai.types import chat as openai_chat_types from pydantic import BaseModel, Field from pipecat.adapters.schemas.tools_schema import ToolsSchema diff --git a/src/pipecat/services/whisper/stt.py b/src/pipecat/services/whisper/stt.py index 27ba743ac2..3865578e17 100644 --- a/src/pipecat/services/whisper/stt.py +++ b/src/pipecat/services/whisper/stt.py @@ -33,7 +33,7 @@ raise Exception(f"Missing module: {e}") try: - import mlx_whisper + import mlx_whisper # noqa: F401 except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use Whisper, you need to `pip install pipecat-ai[mlx-whisper]`.") diff --git a/src/pipecat/sync/base_notifier.py b/src/pipecat/sync/base_notifier.py index 474d50a8b5..fb6e127329 100644 --- a/src/pipecat/sync/base_notifier.py +++ b/src/pipecat/sync/base_notifier.py @@ -8,8 +8,6 @@ import warnings -from pipecat.utils.sync.base_notifier import BaseNotifier - with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( diff --git a/src/pipecat/sync/event_notifier.py b/src/pipecat/sync/event_notifier.py index 2a33cab734..6a6f6abbe7 100644 --- a/src/pipecat/sync/event_notifier.py +++ b/src/pipecat/sync/event_notifier.py @@ -8,8 +8,6 @@ import warnings -from pipecat.utils.sync.event_notifier import EventNotifier - with warnings.catch_warnings(): warnings.simplefilter("always") warnings.warn( diff --git a/src/pipecat/tests/utils.py b/src/pipecat/tests/utils.py index 94b8cb1a41..ac5739829a 100644 --- a/src/pipecat/tests/utils.py +++ b/src/pipecat/tests/utils.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024-2025 Daily +# Copyright (c) 2024-2026, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 31bc3e035b..b5f5a17c76 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -11,6 +11,7 @@ """ import asyncio +import time from typing import Optional from loguru import logger @@ -77,6 +78,11 @@ def __init__(self, params: TransportParams, **kwargs): # Track user speaking state for interruption logic self._user_speaking = False + # Last time a UserSpeakingFrame was pushed. + self._user_speaking_frame_time = 0 + # How often a UserSpeakingFrame should be pushed (value should be + # greater than the audio chunks to have any effect). + self._user_speaking_frame_period = 0.2 # Task to process incoming audio (VAD) and push audio frames downstream # if passthrough is enabled. @@ -423,7 +429,7 @@ async def _audio_task_handler(self): await self._deprecated_run_turn_analyzer(frame, vad_state, previous_vad_state) if vad_state == VADState.SPEAKING: - await self.broadcast_frame(UserSpeakingFrame) + await self._user_currently_speaking() # Push audio downstream if passthrough is set. if self._params.audio_in_passthrough: @@ -444,6 +450,13 @@ async def _audio_task_handler(self): else: await self.push_frame(VADUserStoppedSpeakingFrame()) + async def _user_currently_speaking(self): + """Handle user speaking frame.""" + diff_time = time.time() - self._user_speaking_frame_time + if diff_time >= self._user_speaking_frame_period: + await self.broadcast_frame(UserSpeakingFrame) + self._user_speaking_frame_time = time.time() + # # DEPRECATED. # diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index a19d65b75f..c4f59a61d4 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -403,7 +403,7 @@ def __init__( # Last time a BotSpeakingFrame was pushed. self._bot_speaking_frame_time = 0 # How often a BotSpeakingFrame should be pushed (value should be - # lower than the audio chunks). + # greater than the audio chunks to have any effect). self._bot_speaking_frame_period = 0.2 # Last time the bot actually spoke. self._bot_speech_last_time = 0 @@ -644,8 +644,7 @@ async def _bot_currently_speaking(self): diff_time = time.time() - self._bot_speaking_frame_time if diff_time >= self._bot_speaking_frame_period: - await self._transport.push_frame(BotSpeakingFrame()) - await self._transport.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM) + await self._transport.broadcast_frame(BotSpeakingFrame) self._bot_speaking_frame_time = time.time() self._bot_speech_last_time = time.time() diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 203cbb9fa5..3b652b5170 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -759,7 +759,11 @@ async def join(self): # Increment leave counter if we successfully joined. self._leave_counter += 1 - logger.info(f"Joined {self._room_url}") + participant_id = data.get("participants", {}).get("local", {}).get("id") + meeting_id = data.get("meetingSession", {}).get("id") + logger.info( + f"Joined {self._room_url}. Participant ID: {participant_id}, Meeting ID: {meeting_id}" + ) await self._callbacks.on_joined(data) diff --git a/src/pipecat/transports/daily/utils.py b/src/pipecat/transports/daily/utils.py index bc4c4fa9d8..e5bbca8c71 100644 --- a/src/pipecat/transports/daily/utils.py +++ b/src/pipecat/transports/daily/utils.py @@ -10,11 +10,11 @@ """ import time -from typing import Dict, List, Literal, Optional +from typing import Any, Dict, List, Literal, Optional from urllib.parse import urlparse import aiohttp -from pydantic import BaseModel, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError class DailyRoomSipParams(BaseModel): @@ -77,7 +77,7 @@ class TranscriptionBucketConfig(BaseModel): allow_api_access: bool = False -class DailyRoomProperties(BaseModel, extra="allow"): +class DailyRoomProperties(BaseModel): """Properties for configuring a Daily room. Reference: https://docs.daily.co/reference/rest-api/rooms/create-room#properties @@ -100,6 +100,8 @@ class DailyRoomProperties(BaseModel, extra="allow"): start_video_off: Whether video is off by default. """ + model_config = ConfigDict(extra="allow") + exp: Optional[float] = None enable_chat: bool = False enable_prejoin_ui: bool = False @@ -113,7 +115,7 @@ class DailyRoomProperties(BaseModel, extra="allow"): recordings_bucket: Optional[RecordingsBucketConfig] = None transcription_bucket: Optional[TranscriptionBucketConfig] = None sip: Optional[DailyRoomSipParams] = None - sip_uri: Optional[dict] = None + sip_uri: Optional[Dict[str, Any]] = None start_video_off: bool = False @property @@ -203,7 +205,7 @@ class DailyMeetingTokenProperties(BaseModel): enable_recording: Optional[Literal["cloud", "local", "raw-tracks"]] = None enable_prejoin_ui: Optional[bool] = None start_cloud_recording: Optional[bool] = None - permissions: Optional[dict] = None + permissions: Optional[Dict[str, Any]] = None class DailyMeetingTokenParams(BaseModel): diff --git a/src/pipecat/transports/livekit/transport.py b/src/pipecat/transports/livekit/transport.py index 10a7d8d5d3..ab7325c682 100644 --- a/src/pipecat/transports/livekit/transport.py +++ b/src/pipecat/transports/livekit/transport.py @@ -539,11 +539,14 @@ async def _async_on_track_subscribed( elif track.kind == rtc.TrackKind.KIND_VIDEO: logger.info(f"Video track subscribed: {track.sid} from participant {participant.sid}") self._video_tracks[participant.sid] = track - video_stream = rtc.VideoStream(track) - self._task_manager.create_task( - self._process_video_stream(video_stream, participant.sid), - f"{self}::_process_video_stream", - ) + # Only process video stream if video input is enabled to prevent + # unbounded queue growth when there is no consumer for video frames. + if self._params.video_in_enabled: + video_stream = rtc.VideoStream(track) + self._task_manager.create_task( + self._process_video_stream(video_stream, participant.sid), + f"{self}::_process_video_stream", + ) await self._callbacks.on_video_track_subscribed(participant.sid) async def _async_on_track_unsubscribed( diff --git a/src/pipecat/transports/smallwebrtc/connection.py b/src/pipecat/transports/smallwebrtc/connection.py index 5a5a1450d7..b9b6538f13 100644 --- a/src/pipecat/transports/smallwebrtc/connection.py +++ b/src/pipecat/transports/smallwebrtc/connection.py @@ -14,6 +14,7 @@ import asyncio import json import time +import uuid from typing import Any, List, Literal, Optional, Union from loguru import logger @@ -23,7 +24,6 @@ try: from aiortc import ( - MediaStreamTrack, RTCConfiguration, RTCIceServer, RTCPeerConnection, @@ -278,7 +278,7 @@ def _initialize(self): self._answer: Optional[RTCSessionDescription] = None self._pc = RTCPeerConnection(rtc_config) - self._pc_id = self.name + self._pc_id = f"{self.name}-{uuid.uuid4().hex}" self._setup_listeners() self._data_channel = None self._renegotiation_in_progress = False diff --git a/src/pipecat/transports/websocket/client.py b/src/pipecat/transports/websocket/client.py index 2123a2c4d6..02c891c54b 100644 --- a/src/pipecat/transports/websocket/client.py +++ b/src/pipecat/transports/websocket/client.py @@ -50,6 +50,7 @@ class WebsocketClientParams(TransportParams): """ add_wav_header: bool = True + additional_headers: Optional[dict[str, str]] = None serializer: Optional[FrameSerializer] = None @@ -130,7 +131,11 @@ async def connect(self): return try: - self._websocket = await websocket_connect(uri=self._uri, open_timeout=10) + self._websocket = await websocket_connect( + uri=self._uri, + open_timeout=10, + additional_headers=self._params.additional_headers, + ) self._client_task = self.task_manager.create_task( self._client_task_handler(), f"{self._transport_name}::WebsocketClientSession::_client_task_handler", diff --git a/src/pipecat/turns/mute/__init__.py b/src/pipecat/turns/mute/__init__.py index d452ce19e3..c82b10ae71 100644 --- a/src/pipecat/turns/mute/__init__.py +++ b/src/pipecat/turns/mute/__init__.py @@ -4,10 +4,29 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from pipecat.turns.mute.always_user_mute_strategy import AlwaysUserMuteStrategy -from pipecat.turns.mute.base_user_mute_strategy import BaseUserMuteStrategy -from pipecat.turns.mute.first_speech_user_mute_strategy import FirstSpeechUserMuteStrategy -from pipecat.turns.mute.function_call_user_mute_strategy import FunctionCallUserMuteStrategy -from pipecat.turns.mute.mute_until_first_bot_complete_user_mute_strategy import ( +import warnings + +from pipecat.turns.user_mute.always_user_mute_strategy import AlwaysUserMuteStrategy +from pipecat.turns.user_mute.base_user_mute_strategy import BaseUserMuteStrategy +from pipecat.turns.user_mute.first_speech_user_mute_strategy import FirstSpeechUserMuteStrategy +from pipecat.turns.user_mute.function_call_user_mute_strategy import FunctionCallUserMuteStrategy +from pipecat.turns.user_mute.mute_until_first_bot_complete_user_mute_strategy import ( MuteUntilFirstBotCompleteUserMuteStrategy, ) + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.turns.mute are deprecated. " + "Please use the equivalent types from pipecat.turns.user_mute instead.", + DeprecationWarning, + stacklevel=2, + ) + +__all__ = [ + "AlwaysUserMuteStrategy", + "BaseUserMuteStrategy", + "FirstSpeechUserMuteStrategy", + "FunctionCallUserMuteStrategy", + "MuteUntilFirstBotCompleteUserMuteStrategy", +] diff --git a/src/pipecat/turns/user_idle_controller.py b/src/pipecat/turns/user_idle_controller.py new file mode 100644 index 0000000000..cce35b9cbe --- /dev/null +++ b/src/pipecat/turns/user_idle_controller.py @@ -0,0 +1,173 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""This module defines a controller for managing user idle detection.""" + +import asyncio +from typing import Optional + +from pipecat.frames.frames import ( + BotSpeakingFrame, + Frame, + FunctionCallResultFrame, + FunctionCallsStartedFrame, + UserSpeakingFrame, + UserStartedSpeakingFrame, +) +from pipecat.utils.asyncio.task_manager import BaseTaskManager +from pipecat.utils.base_object import BaseObject + + +class UserIdleController(BaseObject): + """Controller for managing user idle detection. + + This class monitors user activity and triggers an event when the user has been + idle (not speaking) for a configured timeout period. It only starts monitoring + after the first conversation activity and does not trigger while the bot is + speaking or function calls are in progress. + + The controller tracks activity using continuous frames (UserSpeakingFrame and + BotSpeakingFrame) which are emitted repeatedly while speaking is happening, and + state-based tracking for function calls (FunctionCallsStartedFrame and + FunctionCallResultFrame) which are only sent at start and end. + + Event handlers available: + + - on_user_turn_idle: Emitted when the user has been idle for the timeout period. + + Example:: + + @controller.event_handler("on_user_turn_idle") + async def on_user_turn_idle(controller): + # Handle user idle - send reminder, prompt, etc. + ... + """ + + def __init__( + self, + *, + user_idle_timeout: float, + ): + """Initialize the user idle controller. + + Args: + user_idle_timeout: Timeout in seconds before considering the user idle. + """ + super().__init__() + + self._user_idle_timeout = user_idle_timeout + + self._task_manager: Optional[BaseTaskManager] = None + + self._conversation_started = False + self._function_call_in_progress = False + + self.user_idle_event = asyncio.Event() + self.user_idle_task: Optional[asyncio.Task] = None + + self._register_event_handler("on_user_turn_idle", sync=True) + + @property + def task_manager(self) -> BaseTaskManager: + """Returns the configured task manager.""" + if not self._task_manager: + raise RuntimeError(f"{self} user idle controller was not properly setup") + return self._task_manager + + async def setup(self, task_manager: BaseTaskManager): + """Initialize the controller with the given task manager. + + Args: + task_manager: The task manager to be associated with this instance. + """ + self._task_manager = task_manager + + if not self.user_idle_task: + self.user_idle_task = self.task_manager.create_task( + self.user_idle_task_handler(), + f"{self}::user_idle_task_handler", + ) + + async def cleanup(self): + """Cleanup the controller.""" + await super().cleanup() + + if self.user_idle_task: + await self.task_manager.cancel_task(self.user_idle_task) + self.user_idle_task = None + + async def process_frame(self, frame: Frame): + """Process an incoming frame to track user activity state. + + Args: + frame: The frame to be processed. + """ + # Start monitoring on first conversation activity + if not self._conversation_started: + if isinstance(frame, (UserStartedSpeakingFrame, BotSpeakingFrame)): + self._conversation_started = True + self.user_idle_event.set() + else: + return + + # Reset idle timer on continuous activity frames + if isinstance(frame, (UserSpeakingFrame, BotSpeakingFrame)): + await self._handle_activity(frame) + # Track function call state (start/end frames, not continuous) + elif isinstance(frame, FunctionCallsStartedFrame): + await self._handle_function_calls_started(frame) + elif isinstance(frame, FunctionCallResultFrame): + await self._handle_function_call_result(frame) + + async def _handle_activity(self, _: UserSpeakingFrame | BotSpeakingFrame): + """Handle continuous activity frames that should reset the idle timer. + + These frames are emitted continuously while the user or bot is speaking, + so we simply reset the timer whenever we receive them. + + Args: + frame: The activity frame to process. + """ + self.user_idle_event.set() + + async def _handle_function_calls_started(self, _: FunctionCallsStartedFrame): + """Handle function calls started event. + + Function calls can take longer than the timeout, so we track their state + to prevent idle callbacks while they're in progress. + + Args: + frame: The FunctionCallsStartedFrame to process. + """ + self._function_call_in_progress = True + self.user_idle_event.set() + + async def _handle_function_call_result(self, _: FunctionCallResultFrame): + """Handle function call result event. + + Args: + frame: The FunctionCallResultFrame to process. + """ + self._function_call_in_progress = False + self.user_idle_event.set() + + async def user_idle_task_handler(self): + """Monitors for idle timeout and triggers events. + + Runs in a loop until cancelled. The idle timer is reset whenever activity + frames are received (UserSpeakingFrame or BotSpeakingFrame). Function calls + are tracked via state since they only send start/end frames. If no activity + is detected for the configured timeout period and no function call is in + progress, the on_user_turn_idle event is triggered. + """ + while True: + try: + await asyncio.wait_for(self.user_idle_event.wait(), timeout=self._user_idle_timeout) + self.user_idle_event.clear() + except asyncio.TimeoutError: + # Only trigger if conversation has started and no function call is in progress + if self._conversation_started and not self._function_call_in_progress: + await self._call_event_handler("on_user_turn_idle") diff --git a/src/pipecat/turns/user_mute/__init__.py b/src/pipecat/turns/user_mute/__init__.py new file mode 100644 index 0000000000..14c5fea428 --- /dev/null +++ b/src/pipecat/turns/user_mute/__init__.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from .always_user_mute_strategy import AlwaysUserMuteStrategy +from .base_user_mute_strategy import BaseUserMuteStrategy +from .first_speech_user_mute_strategy import FirstSpeechUserMuteStrategy +from .function_call_user_mute_strategy import FunctionCallUserMuteStrategy +from .mute_until_first_bot_complete_user_mute_strategy import ( + MuteUntilFirstBotCompleteUserMuteStrategy, +) + +__all__ = [ + "AlwaysUserMuteStrategy", + "BaseUserMuteStrategy", + "FirstSpeechUserMuteStrategy", + "FunctionCallUserMuteStrategy", + "MuteUntilFirstBotCompleteUserMuteStrategy", +] diff --git a/src/pipecat/turns/mute/always_user_mute_strategy.py b/src/pipecat/turns/user_mute/always_user_mute_strategy.py similarity index 93% rename from src/pipecat/turns/mute/always_user_mute_strategy.py rename to src/pipecat/turns/user_mute/always_user_mute_strategy.py index 584b8165ba..981c6bb74b 100644 --- a/src/pipecat/turns/mute/always_user_mute_strategy.py +++ b/src/pipecat/turns/user_mute/always_user_mute_strategy.py @@ -7,7 +7,7 @@ """User mute strategy that always mutes the user while the bot is speaking.""" from pipecat.frames.frames import BotStartedSpeakingFrame, BotStoppedSpeakingFrame, Frame -from pipecat.turns.mute.base_user_mute_strategy import BaseUserMuteStrategy +from pipecat.turns.user_mute.base_user_mute_strategy import BaseUserMuteStrategy class AlwaysUserMuteStrategy(BaseUserMuteStrategy): diff --git a/src/pipecat/turns/mute/base_user_mute_strategy.py b/src/pipecat/turns/user_mute/base_user_mute_strategy.py similarity index 100% rename from src/pipecat/turns/mute/base_user_mute_strategy.py rename to src/pipecat/turns/user_mute/base_user_mute_strategy.py diff --git a/src/pipecat/turns/mute/first_speech_user_mute_strategy.py b/src/pipecat/turns/user_mute/first_speech_user_mute_strategy.py similarity index 96% rename from src/pipecat/turns/mute/first_speech_user_mute_strategy.py rename to src/pipecat/turns/user_mute/first_speech_user_mute_strategy.py index 1ce4c64f58..9349edd119 100644 --- a/src/pipecat/turns/mute/first_speech_user_mute_strategy.py +++ b/src/pipecat/turns/user_mute/first_speech_user_mute_strategy.py @@ -7,7 +7,7 @@ """User mute strategy that mutes the user only during the bot’s first speech.""" from pipecat.frames.frames import BotStartedSpeakingFrame, BotStoppedSpeakingFrame, Frame -from pipecat.turns.mute.base_user_mute_strategy import BaseUserMuteStrategy +from pipecat.turns.user_mute.base_user_mute_strategy import BaseUserMuteStrategy class FirstSpeechUserMuteStrategy(BaseUserMuteStrategy): diff --git a/src/pipecat/turns/mute/function_call_user_mute_strategy.py b/src/pipecat/turns/user_mute/function_call_user_mute_strategy.py similarity index 95% rename from src/pipecat/turns/mute/function_call_user_mute_strategy.py rename to src/pipecat/turns/user_mute/function_call_user_mute_strategy.py index ef83de271a..f50f31bd46 100644 --- a/src/pipecat/turns/mute/function_call_user_mute_strategy.py +++ b/src/pipecat/turns/user_mute/function_call_user_mute_strategy.py @@ -14,7 +14,7 @@ FunctionCallResultFrame, FunctionCallsStartedFrame, ) -from pipecat.turns.mute.base_user_mute_strategy import BaseUserMuteStrategy +from pipecat.turns.user_mute.base_user_mute_strategy import BaseUserMuteStrategy class FunctionCallUserMuteStrategy(BaseUserMuteStrategy): diff --git a/src/pipecat/turns/mute/mute_until_first_bot_complete_user_mute_strategy.py b/src/pipecat/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.py similarity index 94% rename from src/pipecat/turns/mute/mute_until_first_bot_complete_user_mute_strategy.py rename to src/pipecat/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.py index 6ac5d62a9e..c19499e070 100644 --- a/src/pipecat/turns/mute/mute_until_first_bot_complete_user_mute_strategy.py +++ b/src/pipecat/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.py @@ -7,7 +7,7 @@ """User mute strategy that mutes the user until the bot completes its first speech.""" from pipecat.frames.frames import BotStoppedSpeakingFrame, Frame -from pipecat.turns.mute.base_user_mute_strategy import BaseUserMuteStrategy +from pipecat.turns.user_mute.base_user_mute_strategy import BaseUserMuteStrategy class MuteUntilFirstBotCompleteUserMuteStrategy(BaseUserMuteStrategy): @@ -51,6 +51,5 @@ async def process_frame(self, frame: Frame) -> bool: return not self._first_speech_handled async def _handle_bot_stopped_speaking(self, frame: BotStoppedSpeakingFrame): - self._bot_speaking = False if not self._first_speech_handled: self._first_speech_handled = True diff --git a/src/pipecat/turns/user_start/__init__.py b/src/pipecat/turns/user_start/__init__.py index 3108e1c9aa..fb84f62edb 100644 --- a/src/pipecat/turns/user_start/__init__.py +++ b/src/pipecat/turns/user_start/__init__.py @@ -4,15 +4,17 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from pipecat.turns.user_start.base_user_turn_start_strategy import ( - BaseUserTurnStartStrategy, - UserTurnStartedParams, -) -from pipecat.turns.user_start.external_user_turn_start_strategy import ExternalUserTurnStartStrategy -from pipecat.turns.user_start.min_words_user_turn_start_strategy import ( - MinWordsUserTurnStartStrategy, -) -from pipecat.turns.user_start.transcription_user_turn_start_strategy import ( - TranscriptionUserTurnStartStrategy, -) -from pipecat.turns.user_start.vad_user_turn_start_strategy import VADUserTurnStartStrategy +from .base_user_turn_start_strategy import BaseUserTurnStartStrategy, UserTurnStartedParams +from .external_user_turn_start_strategy import ExternalUserTurnStartStrategy +from .min_words_user_turn_start_strategy import MinWordsUserTurnStartStrategy +from .transcription_user_turn_start_strategy import TranscriptionUserTurnStartStrategy +from .vad_user_turn_start_strategy import VADUserTurnStartStrategy + +__all__ = [ + "BaseUserTurnStartStrategy", + "ExternalUserTurnStartStrategy", + "MinWordsUserTurnStartStrategy", + "TranscriptionUserTurnStartStrategy", + "UserTurnStartedParams", + "VADUserTurnStartStrategy", +] diff --git a/src/pipecat/turns/user_start/min_words_user_turn_start_strategy.py b/src/pipecat/turns/user_start/min_words_user_turn_start_strategy.py index 1f156cef90..57a0ca0d89 100644 --- a/src/pipecat/turns/user_start/min_words_user_turn_start_strategy.py +++ b/src/pipecat/turns/user_start/min_words_user_turn_start_strategy.py @@ -41,12 +41,10 @@ def __init__(self, *, min_words: int, use_interim: bool = True, **kwargs): self._min_words = min_words self._use_interim = use_interim self._bot_speaking = False - self._text = "" async def reset(self): """Reset the strategy to its initial state.""" await super().reset() - self._text = "" self._bot_speaking = False async def process_frame(self, frame: Frame): @@ -67,7 +65,7 @@ async def process_frame(self, frame: Frame): elif isinstance(frame, TranscriptionFrame): await self._handle_transcription(frame) elif isinstance(frame, InterimTranscriptionFrame) and self._use_interim: - await self._handle_interim_transcription(frame) + await self._handle_transcription(frame) async def _handle_bot_started_speaking(self, frame: BotStartedSpeakingFrame): """Handle bot started speaking frame. @@ -89,41 +87,21 @@ async def _handle_bot_stopped_speaking(self, frame: BotStoppedSpeakingFrame): """ self._bot_speaking = False - async def _handle_transcription(self, frame: TranscriptionFrame): + async def _handle_transcription(self, frame: TranscriptionFrame | InterimTranscriptionFrame): """Handle a completed transcription frame and check word count. Args: frame: The transcription frame to be processed. """ - self._text += frame.text - - min_words = self._min_words if self._bot_speaking else 1 - - word_count = len(self._text.split()) - should_trigger = word_count >= min_words - - logger.debug( - f"{self} should_trigger={should_trigger} num_spoken_words={word_count} " - f"min_words={min_words} bot_speaking={self._bot_speaking}" - ) - - if should_trigger: - await self.trigger_user_turn_started() - - async def _handle_interim_transcription(self, frame: InterimTranscriptionFrame): - """Handle an interim transcription frame and check word count. - - Args: - frame: The interim transcription frame to be processed. - """ min_words = self._min_words if self._bot_speaking else 1 word_count = len(frame.text.split()) should_trigger = word_count >= min_words + is_interim = isinstance(frame, InterimTranscriptionFrame) logger.debug( - f"{self} interim=True should_trigger={should_trigger} num_spoken_words={word_count} " - f"min_words={min_words} bot_speaking={self._bot_speaking}" + f"{self} should_trigger={should_trigger} num_spoken_words={word_count} " + f"min_words={min_words} bot_speaking={self._bot_speaking} interim_transcription={is_interim}" ) if should_trigger: diff --git a/src/pipecat/turns/user_stop/__init__.py b/src/pipecat/turns/user_stop/__init__.py index f54e00b7ef..b95a71d150 100644 --- a/src/pipecat/turns/user_stop/__init__.py +++ b/src/pipecat/turns/user_stop/__init__.py @@ -4,14 +4,15 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from pipecat.turns.user_stop.base_user_turn_stop_strategy import ( - BaseUserTurnStopStrategy, - UserTurnStoppedParams, -) -from pipecat.turns.user_stop.external_user_turn_stop_strategy import ExternalUserTurnStopStrategy -from pipecat.turns.user_stop.transcription_user_turn_stop_strategy import ( - TranscriptionUserTurnStopStrategy, -) -from pipecat.turns.user_stop.turn_analyzer_user_turn_stop_strategy import ( - TurnAnalyzerUserTurnStopStrategy, -) +from .base_user_turn_stop_strategy import BaseUserTurnStopStrategy, UserTurnStoppedParams +from .external_user_turn_stop_strategy import ExternalUserTurnStopStrategy +from .transcription_user_turn_stop_strategy import TranscriptionUserTurnStopStrategy +from .turn_analyzer_user_turn_stop_strategy import TurnAnalyzerUserTurnStopStrategy + +__all__ = [ + "BaseUserTurnStopStrategy", + "ExternalUserTurnStopStrategy", + "UserTurnStoppedParams", + "TranscriptionUserTurnStopStrategy", + "TurnAnalyzerUserTurnStopStrategy", +] diff --git a/src/pipecat/turns/user_stop/turn_analyzer_user_turn_stop_strategy.py b/src/pipecat/turns/user_stop/turn_analyzer_user_turn_stop_strategy.py index b21177dc44..1bb2663801 100644 --- a/src/pipecat/turns/user_stop/turn_analyzer_user_turn_stop_strategy.py +++ b/src/pipecat/turns/user_stop/turn_analyzer_user_turn_stop_strategy.py @@ -10,7 +10,6 @@ from typing import Optional from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, EndOfTurnState -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.frames.frames import ( Frame, InputAudioRawFrame, diff --git a/src/pipecat/turns/user_turn_controller.py b/src/pipecat/turns/user_turn_controller.py index a9b267ae0e..05cb46988e 100644 --- a/src/pipecat/turns/user_turn_controller.py +++ b/src/pipecat/turns/user_turn_controller.py @@ -12,6 +12,8 @@ from pipecat.frames.frames import ( Frame, TranscriptionFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, ) @@ -80,7 +82,7 @@ def __init__( self._task_manager: Optional[BaseTaskManager] = None - self._vad_user_speaking = False + self._user_speaking = False self._user_turn = False self._user_turn_stop_timeout_event = asyncio.Event() @@ -146,7 +148,11 @@ async def process_frame(self, frame: Frame): frame: The frame to be processed. """ - if isinstance(frame, VADUserStartedSpeakingFrame): + if isinstance(frame, UserStartedSpeakingFrame): + await self._handle_user_started_speaking(frame) + elif isinstance(frame, UserStoppedSpeakingFrame): + await self._handle_user_stopped_speaking(frame) + elif isinstance(frame, VADUserStartedSpeakingFrame): await self._handle_vad_user_started_speaking(frame) elif isinstance(frame, VADUserStoppedSpeakingFrame): await self._handle_vad_user_stopped_speaking(frame) @@ -179,14 +185,26 @@ async def _cleanup_strategies(self): for s in self._user_turn_strategies.stop or []: await s.cleanup() + async def _handle_user_started_speaking(self, frame: UserStartedSpeakingFrame): + self._user_speaking = True + + # The user started talking, let's reset the user turn timeout. + self._user_turn_stop_timeout_event.set() + + async def _handle_user_stopped_speaking(self, frame: UserStoppedSpeakingFrame): + self._user_speaking = False + + # The user stopped talking, let's reset the user turn timeout. + self._user_turn_stop_timeout_event.set() + async def _handle_vad_user_started_speaking(self, frame: VADUserStartedSpeakingFrame): - self._vad_user_speaking = True + self._user_speaking = True # The user started talking, let's reset the user turn timeout. self._user_turn_stop_timeout_event.set() async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame): - self._vad_user_speaking = False + self._user_speaking = False # The user stopped talking, let's reset the user turn timeout. self._user_turn_stop_timeout_event.set() @@ -233,6 +251,10 @@ async def _trigger_user_turn_start( self._user_turn = True self._user_turn_stop_timeout_event.set() + # Reset all user turn start strategies to start fresh. + for s in self._user_turn_strategies.start or []: + await s.reset() + await self._call_event_handler("on_user_turn_started", strategy, params) async def _trigger_user_turn_stop( @@ -260,7 +282,7 @@ async def _user_turn_stop_timeout_task_handler(self): ) self._user_turn_stop_timeout_event.clear() except asyncio.TimeoutError: - if self._user_turn and not self._vad_user_speaking: + if self._user_turn and not self._user_speaking: await self._call_event_handler("on_user_turn_stop_timeout") await self._trigger_user_turn_stop( None, UserTurnStoppedParams(enable_user_speaking_frames=True) diff --git a/src/pipecat/turns/user_turn_processor.py b/src/pipecat/turns/user_turn_processor.py index 79e4664a09..6c771811e0 100644 --- a/src/pipecat/turns/user_turn_processor.py +++ b/src/pipecat/turns/user_turn_processor.py @@ -19,6 +19,7 @@ UserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.turns.user_idle_controller import UserIdleController from pipecat.turns.user_start import BaseUserTurnStartStrategy, UserTurnStartedParams from pipecat.turns.user_stop import BaseUserTurnStopStrategy, UserTurnStoppedParams from pipecat.turns.user_turn_controller import UserTurnController @@ -38,6 +39,7 @@ class UserTurnProcessor(FrameProcessor): - on_user_turn_started: Emitted when a user turn starts. - on_user_turn_stopped: Emitted when a user turn stops. - on_user_turn_stop_timeout: Emitted if no stop strategy triggers before timeout. + - on_user_turn_idle: Emitted when the user has been idle for the configured timeout. Example:: @@ -53,6 +55,10 @@ async def on_user_turn_stopped(processor, strategy: BaseUserTurnStopStrategy): async def on_user_turn_stop_timeout(processor): ... + @processor.event_handler("on_user_turn_idle") + async def on_user_turn_idle(processor): + ... + """ def __init__( @@ -60,6 +66,7 @@ def __init__( *, user_turn_strategies: Optional[UserTurnStrategies] = None, user_turn_stop_timeout: float = 5.0, + user_idle_timeout: Optional[float] = None, **kwargs, ): """Initialize the user turn processor. @@ -68,6 +75,10 @@ def __init__( user_turn_strategies: Configured strategies for starting and stopping user turns. user_turn_stop_timeout: Timeout in seconds to automatically stop a user turn if no activity is detected. + user_idle_timeout: Optional timeout in seconds for detecting user idle state. + If set, the processor will emit an `on_user_turn_idle` event when the user + has been idle (not speaking) for this duration. Set to None to disable + idle detection. **kwargs: Additional keyword arguments. """ super().__init__(**kwargs) @@ -75,6 +86,7 @@ def __init__( self._register_event_handler("on_user_turn_started") self._register_event_handler("on_user_turn_stopped") self._register_event_handler("on_user_turn_stop_timeout") + self._register_event_handler("on_user_turn_idle") self._user_turn_controller = UserTurnController( user_turn_strategies=user_turn_strategies or UserTurnStrategies(), @@ -92,6 +104,14 @@ def __init__( "on_user_turn_stop_timeout", self._on_user_turn_stop_timeout ) + # Optional user idle controller + self._user_idle_controller: Optional[UserIdleController] = None + if user_idle_timeout: + self._user_idle_controller = UserIdleController(user_idle_timeout=user_idle_timeout) + self._user_idle_controller.add_event_handler( + "on_user_turn_idle", self._on_user_turn_idle + ) + async def cleanup(self): """Clean up processor resources.""" await super().cleanup() @@ -129,9 +149,15 @@ async def process_frame(self, frame: Frame, direction: FrameDirection): await self._user_turn_controller.process_frame(frame) + if self._user_idle_controller: + await self._user_idle_controller.process_frame(frame) + async def _start(self, frame: StartFrame): await self._user_turn_controller.setup(self.task_manager) + if self._user_idle_controller: + await self._user_idle_controller.setup(self.task_manager) + async def _stop(self, frame: EndFrame): await self._cleanup() @@ -141,6 +167,9 @@ async def _cancel(self, frame: CancelFrame): async def _cleanup(self): await self._user_turn_controller.cleanup() + if self._user_idle_controller: + await self._user_idle_controller.cleanup() + async def _on_push_frame( self, controller, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM ): @@ -180,3 +209,6 @@ async def _on_user_turn_stopped( async def _on_user_turn_stop_timeout(self, controller): await self._call_event_handler("on_user_turn_stop_timeout") + + async def _on_user_turn_idle(self, controller): + await self._call_event_handler("on_user_turn_idle") diff --git a/src/pipecat/utils/text/skip_tags_aggregator.py b/src/pipecat/utils/text/skip_tags_aggregator.py index 1212bd34da..4232efd7dd 100644 --- a/src/pipecat/utils/text/skip_tags_aggregator.py +++ b/src/pipecat/utils/text/skip_tags_aggregator.py @@ -14,7 +14,7 @@ from typing import AsyncIterator, Optional, Sequence from pipecat.utils.string import StartEndTags, parse_start_end_tags -from pipecat.utils.text.base_text_aggregator import Aggregation, AggregationType +from pipecat.utils.text.base_text_aggregator import Aggregation from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator diff --git a/tests/integration/test_integration_unified_function_calling.py b/tests/integration/test_integration_unified_function_calling.py index d017c55acf..552c839a5e 100644 --- a/tests/integration/test_integration_unified_function_calling.py +++ b/tests/integration/test_integration_unified_function_calling.py @@ -5,7 +5,6 @@ # import os -from unittest.mock import AsyncMock import pytest from dotenv import load_dotenv diff --git a/tests/test_context_aggregators_universal.py b/tests/test_context_aggregators_universal.py index 6aa38d362f..3cb1407f13 100644 --- a/tests/test_context_aggregators_universal.py +++ b/tests/test_context_aggregators_universal.py @@ -40,7 +40,7 @@ LLMUserAggregatorParams, ) from pipecat.tests.utils import SleepFrame, run_test -from pipecat.turns.mute import FirstSpeechUserMuteStrategy, FunctionCallUserMuteStrategy +from pipecat.turns.user_mute import FirstSpeechUserMuteStrategy, FunctionCallUserMuteStrategy from pipecat.turns.user_stop import TranscriptionUserTurnStopStrategy from pipecat.turns.user_turn_strategies import UserTurnStrategies diff --git a/tests/test_get_llm_invocation_params.py b/tests/test_get_llm_invocation_params.py index ad1437227e..c93275b67b 100644 --- a/tests/test_get_llm_invocation_params.py +++ b/tests/test_get_llm_invocation_params.py @@ -43,7 +43,6 @@ import unittest from google.genai.types import Content, Part -from openai.types.chat import ChatCompletionMessage from pipecat.adapters.services.anthropic_adapter import AnthropicLLMAdapter from pipecat.adapters.services.bedrock_adapter import AWSBedrockLLMAdapter @@ -51,7 +50,6 @@ from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter from pipecat.processors.aggregators.llm_context import ( LLMContext, - LLMSpecificMessage, LLMStandardMessage, ) diff --git a/tests/test_krisp_sdk_manager.py b/tests/test_krisp_sdk_manager.py index a98c97f09e..7c774ab85e 100644 --- a/tests/test_krisp_sdk_manager.py +++ b/tests/test_krisp_sdk_manager.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024-2025 Daily +# Copyright (c) 2024-2026, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/tests/test_krisp_viva_filter.py b/tests/test_krisp_viva_filter.py index c8f1a23c0d..a788ab1316 100644 --- a/tests/test_krisp_viva_filter.py +++ b/tests/test_krisp_viva_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024-2025 Daily +# Copyright (c) 2024-2026, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -9,7 +9,7 @@ import sys import tempfile import unittest -from unittest.mock import AsyncMock, MagicMock, Mock, patch +from unittest.mock import MagicMock, patch import numpy as np diff --git a/tests/test_livekit_transport.py b/tests/test_livekit_transport.py new file mode 100644 index 0000000000..472d60cbae --- /dev/null +++ b/tests/test_livekit_transport.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Tests for LiveKit transport video stream handling. + +Regression tests for issue #3116: Memory leak when video_in_enabled=False +but video tracks are subscribed. The fix ensures video stream processing +only starts when there is a consumer for the frames. +""" + +import unittest +from unittest.mock import AsyncMock, MagicMock, patch + +try: + from livekit import rtc + + from pipecat.transports.livekit.transport import ( + LiveKitCallbacks, + LiveKitParams, + LiveKitTransportClient, + ) + + LIVEKIT_AVAILABLE = True +except ImportError: + LIVEKIT_AVAILABLE = False + + +@unittest.skipUnless(LIVEKIT_AVAILABLE, "livekit package not installed") +class TestLiveKitVideoStreamMemoryLeak(unittest.IsolatedAsyncioTestCase): + """Regression tests for video queue memory leak (#3116). + + The bug: When video_in_enabled=False, subscribing to a video track would + start a producer that fills _video_queue, but no consumer would drain it, + causing unbounded memory growth (~3GB/min). + + The fix: Only start video stream processing when video_in_enabled=True. + """ + + def _create_client(self, video_in_enabled: bool) -> LiveKitTransportClient: + """Create a client with the specified video input setting.""" + params = LiveKitParams(video_in_enabled=video_in_enabled) + callbacks = LiveKitCallbacks( + on_connected=AsyncMock(), + on_disconnected=AsyncMock(), + on_before_disconnect=AsyncMock(), + on_participant_connected=AsyncMock(), + on_participant_disconnected=AsyncMock(), + on_audio_track_subscribed=AsyncMock(), + on_audio_track_unsubscribed=AsyncMock(), + on_video_track_subscribed=AsyncMock(), + on_video_track_unsubscribed=AsyncMock(), + on_data_received=AsyncMock(), + on_first_participant_joined=AsyncMock(), + ) + client = LiveKitTransportClient( + url="wss://test.livekit.cloud", + token="test-token", + room_name="test-room", + params=params, + callbacks=callbacks, + transport_name="test-transport", + ) + client._task_manager = MagicMock() + return client + + def _create_mock_video_track(self): + """Create a mock video track subscription event.""" + track = MagicMock() + track.kind = rtc.TrackKind.KIND_VIDEO + track.sid = "video-track-123" + publication = MagicMock() + participant = MagicMock() + participant.sid = "participant-456" + return track, publication, participant + + async def test_disabled_video_input_does_not_start_queue_producer(self): + """When video input is disabled, no producer should fill the queue. + + This prevents the memory leak where frames accumulate with no consumer. + """ + client = self._create_client(video_in_enabled=False) + track, publication, participant = self._create_mock_video_track() + + await client._async_on_track_subscribed(track, publication, participant) + + # Verify no video processing task was started + task_names = [call[0][1] for call in client._task_manager.create_task.call_args_list] + video_tasks = [name for name in task_names if "video" in name.lower()] + self.assertEqual(video_tasks, [], "No video processing task should be started") + + # Queue should remain empty + self.assertEqual(client._video_queue.qsize(), 0) + + # Track metadata should still be recorded + self.assertIn(participant.sid, client._video_tracks) + + # Callback should still fire for user code + client._callbacks.on_video_track_subscribed.assert_called_once() + + async def test_enabled_video_input_starts_queue_producer(self): + """When video input is enabled, the producer should start.""" + client = self._create_client(video_in_enabled=True) + track, publication, participant = self._create_mock_video_track() + + with patch.object(rtc, "VideoStream"): + await client._async_on_track_subscribed(track, publication, participant) + + # Verify video processing task was started + task_names = [call[0][1] for call in client._task_manager.create_task.call_args_list] + video_tasks = [name for name in task_names if "video" in name.lower()] + self.assertEqual(len(video_tasks), 1, "Video processing task should be started") + + # Track metadata should be recorded + self.assertIn(participant.sid, client._video_tracks) + + # Callback should fire + client._callbacks.on_video_track_subscribed.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_rnnoise_filter.py b/tests/test_rnnoise_filter.py index 9e53b5b473..db3ac7825b 100644 --- a/tests/test_rnnoise_filter.py +++ b/tests/test_rnnoise_filter.py @@ -5,7 +5,7 @@ # import unittest -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock import numpy as np diff --git a/tests/test_rnnoise_resampling.py b/tests/test_rnnoise_resampling.py index c69419d2e4..060c9e802f 100644 --- a/tests/test_rnnoise_resampling.py +++ b/tests/test_rnnoise_resampling.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import sys import unittest from unittest.mock import MagicMock, patch diff --git a/tests/test_stt_mute_filter.py b/tests/test_stt_mute_filter.py index 459060b86c..39631d7e8f 100644 --- a/tests/test_stt_mute_filter.py +++ b/tests/test_stt_mute_filter.py @@ -10,7 +10,6 @@ BotStartedSpeakingFrame, BotStoppedSpeakingFrame, FunctionCallFromLLM, - FunctionCallInProgressFrame, FunctionCallResultFrame, FunctionCallsStartedFrame, InputAudioRawFrame, diff --git a/tests/test_user_idle_controller.py b/tests/test_user_idle_controller.py new file mode 100644 index 0000000000..5de665353e --- /dev/null +++ b/tests/test_user_idle_controller.py @@ -0,0 +1,216 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import unittest + +from pipecat.frames.frames import ( + BotSpeakingFrame, + FunctionCallResultFrame, + FunctionCallsStartedFrame, + UserSpeakingFrame, + UserStartedSpeakingFrame, +) +from pipecat.turns.user_idle_controller import UserIdleController +from pipecat.utils.asyncio.task_manager import TaskManager, TaskManagerParams + +USER_IDLE_TIMEOUT = 0.2 + + +class TestUserIdleController(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.task_manager = TaskManager() + self.task_manager.setup(TaskManagerParams(loop=asyncio.get_running_loop())) + + async def test_basic_idle_detection(self): + """Test that idle event is triggered after timeout when no activity.""" + controller = UserIdleController(user_idle_timeout=USER_IDLE_TIMEOUT) + await controller.setup(self.task_manager) + + idle_triggered = False + + @controller.event_handler("on_user_turn_idle") + async def on_user_turn_idle(controller): + nonlocal idle_triggered + idle_triggered = True + + # Start conversation + await controller.process_frame(UserStartedSpeakingFrame()) + + # Wait for idle timeout + await asyncio.sleep(USER_IDLE_TIMEOUT + 0.1) + + self.assertTrue(idle_triggered) + + await controller.cleanup() + + async def test_user_speaking_resets_idle_timer(self): + """Test that continuous UserSpeakingFrame frames reset the idle timer.""" + controller = UserIdleController(user_idle_timeout=USER_IDLE_TIMEOUT) + await controller.setup(self.task_manager) + + idle_triggered = False + + @controller.event_handler("on_user_turn_idle") + async def on_user_turn_idle(controller): + nonlocal idle_triggered + idle_triggered = True + + # Start conversation + await controller.process_frame(UserStartedSpeakingFrame()) + + # Send UserSpeakingFrame continuously to reset timer + for _ in range(5): + await asyncio.sleep(USER_IDLE_TIMEOUT * 0.5) # 50% of timeout period + await controller.process_frame(UserSpeakingFrame()) + + self.assertFalse(idle_triggered) + + await controller.cleanup() + + async def test_bot_speaking_resets_idle_timer(self): + """Test that BotSpeakingFrame frames reset the idle timer.""" + controller = UserIdleController(user_idle_timeout=USER_IDLE_TIMEOUT) + await controller.setup(self.task_manager) + + idle_triggered = False + + @controller.event_handler("on_user_turn_idle") + async def on_user_turn_idle(controller): + nonlocal idle_triggered + idle_triggered = True + + # Start conversation + await controller.process_frame(UserStartedSpeakingFrame()) + + # Bot speaking should reset timer + for _ in range(5): + await asyncio.sleep(USER_IDLE_TIMEOUT * 0.6) # 60% of timeout + await controller.process_frame(BotSpeakingFrame()) + + self.assertFalse(idle_triggered) + + await controller.cleanup() + + async def test_function_call_prevents_idle(self): + """Test that function calls in progress prevent idle event.""" + controller = UserIdleController(user_idle_timeout=USER_IDLE_TIMEOUT) + await controller.setup(self.task_manager) + + idle_triggered = False + + @controller.event_handler("on_user_turn_idle") + async def on_user_turn_idle(controller): + nonlocal idle_triggered + idle_triggered = True + + # Start conversation + await controller.process_frame(UserStartedSpeakingFrame()) + + # Start function call + await controller.process_frame(FunctionCallsStartedFrame(function_calls=[])) + + # Wait longer than idle timeout + await asyncio.sleep(USER_IDLE_TIMEOUT + 0.1) + + # Should not trigger idle because function call is in progress + self.assertFalse(idle_triggered) + + # Complete function call + await controller.process_frame( + FunctionCallResultFrame( + function_name="test", + tool_call_id="123", + arguments={}, + result=None, + run_llm=False, + ) + ) + + # Now idle should trigger + await asyncio.sleep(USER_IDLE_TIMEOUT + 0.1) + self.assertTrue(idle_triggered) + + await controller.cleanup() + + async def test_no_idle_before_conversation_starts(self): + """Test that idle monitoring doesn't start before first conversation activity.""" + controller = UserIdleController(user_idle_timeout=USER_IDLE_TIMEOUT) + await controller.setup(self.task_manager) + + idle_triggered = False + + @controller.event_handler("on_user_turn_idle") + async def on_user_turn_idle(controller): + nonlocal idle_triggered + idle_triggered = True + + # Wait without starting conversation + await asyncio.sleep(USER_IDLE_TIMEOUT + 0.1) + + self.assertFalse(idle_triggered) + + await controller.cleanup() + + async def test_idle_starts_with_bot_speaking(self): + """Test that monitoring starts with BotSpeakingFrame, not just user speech.""" + controller = UserIdleController(user_idle_timeout=USER_IDLE_TIMEOUT) + await controller.setup(self.task_manager) + + idle_triggered = False + + @controller.event_handler("on_user_turn_idle") + async def on_user_turn_idle(controller): + nonlocal idle_triggered + idle_triggered = True + + # Start conversation with bot speaking + await controller.process_frame(BotSpeakingFrame()) + + # Wait for idle timeout + await asyncio.sleep(USER_IDLE_TIMEOUT + 0.1) + + self.assertTrue(idle_triggered) + + await controller.cleanup() + + async def test_multiple_idle_events(self): + """Test that idle event can trigger multiple times.""" + controller = UserIdleController(user_idle_timeout=USER_IDLE_TIMEOUT) + await controller.setup(self.task_manager) + + idle_count = 0 + + @controller.event_handler("on_user_turn_idle") + async def on_user_turn_idle(controller): + nonlocal idle_count + idle_count += 1 + + # Start conversation + await controller.process_frame(UserStartedSpeakingFrame()) + + # First idle + await asyncio.sleep(USER_IDLE_TIMEOUT + 0.1) + first_count = idle_count + self.assertGreaterEqual(first_count, 1) + + # Second idle + await asyncio.sleep(USER_IDLE_TIMEOUT + 0.1) + second_count = idle_count + self.assertGreater(second_count, first_count) + + # User activity resets timer + await controller.process_frame(UserSpeakingFrame()) + + # Give a moment for the timer to reset + await asyncio.sleep(0.1) + + # Third idle + await asyncio.sleep(USER_IDLE_TIMEOUT + 0.1) + third_count = idle_count + self.assertGreater(third_count, second_count) + + await controller.cleanup() diff --git a/tests/test_user_mute_strategy.py b/tests/test_user_mute_strategy.py index 26a7af1cc7..d8261877e5 100644 --- a/tests/test_user_mute_strategy.py +++ b/tests/test_user_mute_strategy.py @@ -15,7 +15,7 @@ FunctionCallsStartedFrame, InterruptionFrame, ) -from pipecat.turns.mute import ( +from pipecat.turns.user_mute import ( AlwaysUserMuteStrategy, FirstSpeechUserMuteStrategy, FunctionCallUserMuteStrategy, diff --git a/tests/test_user_turn_controller.py b/tests/test_user_turn_controller.py index 6762bd2192..d8f6425946 100644 --- a/tests/test_user_turn_controller.py +++ b/tests/test_user_turn_controller.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024-2026 Daily +# Copyright (c) 2024-2026, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -8,12 +8,18 @@ import unittest from pipecat.frames.frames import ( + BotStartedSpeakingFrame, TranscriptionFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, ) +from pipecat.turns.user_start.min_words_user_turn_start_strategy import ( + MinWordsUserTurnStartStrategy, +) from pipecat.turns.user_turn_controller import UserTurnController -from pipecat.turns.user_turn_strategies import UserTurnStrategies +from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies, UserTurnStrategies from pipecat.utils.asyncio.task_manager import TaskManager, TaskManagerParams USER_TURN_STOP_TIMEOUT = 0.2 @@ -56,6 +62,44 @@ async def on_user_turn_stopped(controller, strategy, params): self.assertTrue(should_start) self.assertTrue(should_stop) + async def test_user_turn_start_reset(self): + controller = UserTurnController( + user_turn_strategies=UserTurnStrategies( + start=[MinWordsUserTurnStartStrategy(min_words=3)] + ), + user_turn_stop_timeout=USER_TURN_STOP_TIMEOUT, + ) + + await controller.setup(self.task_manager) + + should_start = 0 + + @controller.event_handler("on_user_turn_started") + async def on_user_turn_started(controller, strategy, params): + nonlocal should_start + should_start += 1 + + await controller.process_frame(BotStartedSpeakingFrame()) + await controller.process_frame(TranscriptionFrame(text="One", user_id="cat", timestamp="")) + self.assertEqual(should_start, 0) + + await controller.process_frame( + TranscriptionFrame(text="One two three!", user_id="cat", timestamp="") + ) + self.assertEqual(should_start, 1) + + # Trigger user stop turn so we can trigger user start turn again. + await asyncio.sleep(USER_TURN_STOP_TIMEOUT + 0.1) + + await controller.process_frame(BotStartedSpeakingFrame()) + await controller.process_frame(TranscriptionFrame(text="Hi!", user_id="cat", timestamp="")) + self.assertEqual(should_start, 1) + + await controller.process_frame( + TranscriptionFrame(text="How are you?", user_id="cat", timestamp="") + ) + self.assertEqual(should_start, 2) + async def test_user_turn_stop_timeout_no_transcription(self): controller = UserTurnController( user_turn_strategies=UserTurnStrategies(), @@ -96,3 +140,53 @@ async def on_user_turn_stop_timeout(controller): self.assertTrue(should_start) self.assertTrue(should_stop) self.assertTrue(timeout) + + async def test_external_user_turn_strategies_no_timeout_while_speaking(self): + """Test that timeout does not trigger when user is still speaking with external strategies.""" + controller = UserTurnController( + user_turn_strategies=ExternalUserTurnStrategies(), + user_turn_stop_timeout=USER_TURN_STOP_TIMEOUT, + ) + + await controller.setup(self.task_manager) + + should_start = None + should_stop = None + timeout = None + + @controller.event_handler("on_user_turn_started") + async def on_user_turn_started(controller, strategy, params): + nonlocal should_start + should_start = True + + @controller.event_handler("on_user_turn_stopped") + async def on_user_turn_stopped(controller, strategy, params): + nonlocal should_stop + should_stop = True + + @controller.event_handler("on_user_turn_stop_timeout") + async def on_user_turn_stop_timeout(controller): + nonlocal timeout + timeout = True + + # Simulate external service (like Deepgram Flux) broadcasting UserStartedSpeakingFrame + await controller.process_frame(UserStartedSpeakingFrame()) + self.assertTrue(should_start) + self.assertFalse(should_stop) + self.assertFalse(timeout) + + # User is still speaking, timeout should not trigger + await asyncio.sleep(USER_TURN_STOP_TIMEOUT + 0.1) + self.assertTrue(should_start) + self.assertFalse(should_stop) + self.assertFalse(timeout) + + # Now external service broadcasts UserStoppedSpeakingFrame + await controller.process_frame(UserStoppedSpeakingFrame()) + + # But no transcription, so timeout should trigger + await asyncio.sleep(USER_TURN_STOP_TIMEOUT + 0.1) + + self.assertTrue(should_start) + self.assertTrue(should_stop) + self.assertTrue(timeout) diff --git a/tests/test_user_turn_start_strategy.py b/tests/test_user_turn_start_strategy.py index 51856ff9ca..dabd4593b1 100644 --- a/tests/test_user_turn_start_strategy.py +++ b/tests/test_user_turn_start_strategy.py @@ -38,7 +38,7 @@ async def on_user_turn_started(strategy, params): self.assertFalse(should_start) await strategy.process_frame( - TranscriptionFrame(text=" there!", user_id="cat", timestamp="") + TranscriptionFrame(text="Hello there!", user_id="cat", timestamp="") ) self.assertTrue(should_start) @@ -55,6 +55,26 @@ async def on_user_turn_started(strategy, params): ) self.assertTrue(should_start) + async def test_bot_speaking_singlw_words(self): + strategy = MinWordsUserTurnStartStrategy(min_words=3) + + should_start = None + + @strategy.event_handler("on_user_turn_started") + async def on_user_turn_started(strategy, params): + nonlocal should_start + should_start = True + + await strategy.process_frame(BotStartedSpeakingFrame()) + await strategy.process_frame(TranscriptionFrame(text="One", user_id="cat", timestamp="")) + self.assertFalse(should_start) + + await strategy.process_frame(TranscriptionFrame(text="Two", user_id="cat", timestamp="")) + self.assertFalse(should_start) + + await strategy.process_frame(TranscriptionFrame(text="Three", user_id="cat", timestamp="")) + self.assertFalse(should_start) + async def test_bot_speaking_interim_transcriptions(self): strategy = MinWordsUserTurnStartStrategy(min_words=2) diff --git a/uv.lock b/uv.lock index a9b3d2dd49..45c1f892f9 100644 --- a/uv.lock +++ b/uv.lock @@ -629,6 +629,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/c2/80633736cd183ee4a62107413def345f7e6e3c01563dbca1417363cf957e/build-1.2.2.post1-py3-none-any.whl", hash = "sha256:1d61c0887fa860c01971625baae8bdd338e517b836a2f70dd1f7aa3a6b2fc5b5", size = 22950, upload-time = "2024-10-06T17:22:23.299Z" }, ] +[[package]] +name = "camb-sdk" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "pydantic" }, + { name = "typing-extensions" }, + { name = "websocket-client" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/44/b4161f5da381f1b0d35a1abc7d69588ff24af6dacc9f13bccfb6f9889c46/camb_sdk-1.5.4.tar.gz", hash = "sha256:e882fb1b32aa45243ead5a558fed4e4e7ff8542af9f1802565a5fb4b7114fb5c", size = 83074, upload-time = "2026-01-12T20:19:34.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/c9/f718b6028d6e457391d1a6a8f93ae6073055731d77279ef097734b84bdcb/camb_sdk-1.5.4-py3-none-any.whl", hash = "sha256:b9ad43fddc0d749dd522839a06136df89d109510f57e68102414a8c0078e496a", size = 152143, upload-time = "2026-01-12T20:19:31.769Z" }, +] + [[package]] name = "cartesia" version = "2.0.17" @@ -1430,7 +1446,7 @@ wheels = [ [[package]] name = "fastapi" -version = "0.121.3" +version = "0.127.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -1438,9 +1454,9 @@ dependencies = [ { name = "starlette" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/80/f0/086c442c6516195786131b8ca70488c6ef11d2f2e33c9a893576b2b0d3f7/fastapi-0.121.3.tar.gz", hash = "sha256:0055bc24fe53e56a40e9e0ad1ae2baa81622c406e548e501e717634e2dfbc40b", size = 344501, upload-time = "2025-11-19T16:53:39.243Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8a/6b9ba6eb8ff3817caae83120495965d9e70afb4d6348cb120e464ee199f4/fastapi-0.127.1.tar.gz", hash = "sha256:946a87ee5d931883b562b6bada787d6c8178becee2683cb3f9b980d593206359", size = 391876, upload-time = "2025-12-26T13:04:47.075Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/b6/4f620d7720fc0a754c8c1b7501d73777f6ba43b57c8ab99671f4d7441eb8/fastapi-0.121.3-py3-none-any.whl", hash = "sha256:0c78fc87587fcd910ca1bbf5bc8ba37b80e119b388a7206b39f0ecc95ebf53e9", size = 109801, upload-time = "2025-11-19T16:53:37.918Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f3/a6858d147ed2645c095d11dc2440f94a5f1cd8f4df888e3377e6b5281a0f/fastapi-0.127.1-py3-none-any.whl", hash = "sha256:31d670a4f9373cc6d7994420f98e4dc46ea693145207abc39696746c83a44430", size = 112332, upload-time = "2025-12-26T13:04:45.329Z" }, ] [package.optional-dependencies] @@ -1997,6 +2013,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/6a/33d1702184d94106d3cdd7bfb788e19723206fce152e303473ca3b946c7b/greenlet-3.3.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6f8496d434d5cb2dce025773ba5597f71f5410ae499d5dd9533e0653258cdb3d", size = 273658, upload-time = "2025-12-04T14:23:37.494Z" }, { url = "https://files.pythonhosted.org/packages/d6/b7/2b5805bbf1907c26e434f4e448cd8b696a0b71725204fa21a211ff0c04a7/greenlet-3.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b96dc7eef78fd404e022e165ec55327f935b9b52ff355b067eb4a0267fc1cffb", size = 574810, upload-time = "2025-12-04T14:50:04.154Z" }, { url = "https://files.pythonhosted.org/packages/94/38/343242ec12eddf3d8458c73f555c084359883d4ddc674240d9e61ec51fd6/greenlet-3.3.0-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73631cd5cccbcfe63e3f9492aaa664d278fda0ce5c3d43aeda8e77317e38efbd", size = 586248, upload-time = "2025-12-04T14:57:39.35Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d0/0ae86792fb212e4384041e0ef8e7bc66f59a54912ce407d26a966ed2914d/greenlet-3.3.0-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b299a0cb979f5d7197442dccc3aee67fce53500cd88951b7e6c35575701c980b", size = 597403, upload-time = "2025-12-04T15:07:10.831Z" }, { url = "https://files.pythonhosted.org/packages/b6/a8/15d0aa26c0036a15d2659175af00954aaaa5d0d66ba538345bd88013b4d7/greenlet-3.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dee147740789a4632cace364816046e43310b59ff8fb79833ab043aefa72fd5", size = 586910, upload-time = "2025-12-04T14:25:59.705Z" }, { url = "https://files.pythonhosted.org/packages/e1/9b/68d5e3b7ccaba3907e5532cf8b9bf16f9ef5056a008f195a367db0ff32db/greenlet-3.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:39b28e339fc3c348427560494e28d8a6f3561c8d2bcf7d706e1c624ed8d822b9", size = 1547206, upload-time = "2025-12-04T15:04:21.027Z" }, { url = "https://files.pythonhosted.org/packages/66/bd/e3086ccedc61e49f91e2cfb5ffad9d8d62e5dc85e512a6200f096875b60c/greenlet-3.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b3c374782c2935cc63b2a27ba8708471de4ad1abaa862ffdb1ef45a643ddbb7d", size = 1613359, upload-time = "2025-12-04T14:27:26.548Z" }, @@ -2004,6 +2021,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/cb/48e964c452ca2b92175a9b2dca037a553036cb053ba69e284650ce755f13/greenlet-3.3.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e29f3018580e8412d6aaf5641bb7745d38c85228dacf51a73bd4e26ddf2a6a8e", size = 274908, upload-time = "2025-12-04T14:23:26.435Z" }, { url = "https://files.pythonhosted.org/packages/28/da/38d7bff4d0277b594ec557f479d65272a893f1f2a716cad91efeb8680953/greenlet-3.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a687205fb22794e838f947e2194c0566d3812966b41c78709554aa883183fb62", size = 577113, upload-time = "2025-12-04T14:50:05.493Z" }, { url = "https://files.pythonhosted.org/packages/3c/f2/89c5eb0faddc3ff014f1c04467d67dee0d1d334ab81fadbf3744847f8a8a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4243050a88ba61842186cb9e63c7dfa677ec146160b0efd73b855a3d9c7fcf32", size = 590338, upload-time = "2025-12-04T14:57:41.136Z" }, + { url = "https://files.pythonhosted.org/packages/80/d7/db0a5085035d05134f8c089643da2b44cc9b80647c39e93129c5ef170d8f/greenlet-3.3.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:670d0f94cd302d81796e37299bcd04b95d62403883b24225c6b5271466612f45", size = 601098, upload-time = "2025-12-04T15:07:11.898Z" }, { url = "https://files.pythonhosted.org/packages/dc/a6/e959a127b630a58e23529972dbc868c107f9d583b5a9f878fb858c46bc1a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb3a8ec3db4a3b0eb8a3c25436c2d49e3505821802074969db017b87bc6a948", size = 590206, upload-time = "2025-12-04T14:26:01.254Z" }, { url = "https://files.pythonhosted.org/packages/48/60/29035719feb91798693023608447283b266b12efc576ed013dd9442364bb/greenlet-3.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2de5a0b09eab81fc6a382791b995b1ccf2b172a9fec934747a7a23d2ff291794", size = 1550668, upload-time = "2025-12-04T15:04:22.439Z" }, { url = "https://files.pythonhosted.org/packages/0a/5f/783a23754b691bfa86bd72c3033aa107490deac9b2ef190837b860996c9f/greenlet-3.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4449a736606bd30f27f8e1ff4678ee193bc47f6ca810d705981cfffd6ce0d8c5", size = 1615483, upload-time = "2025-12-04T14:27:28.083Z" }, @@ -2011,6 +2029,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/0a/a3871375c7b9727edaeeea994bfff7c63ff7804c9829c19309ba2e058807/greenlet-3.3.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:b01548f6e0b9e9784a2c99c5651e5dc89ffcbe870bc5fb2e5ef864e9cc6b5dcb", size = 276379, upload-time = "2025-12-04T14:23:30.498Z" }, { url = "https://files.pythonhosted.org/packages/43/ab/7ebfe34dce8b87be0d11dae91acbf76f7b8246bf9d6b319c741f99fa59c6/greenlet-3.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:349345b770dc88f81506c6861d22a6ccd422207829d2c854ae2af8025af303e3", size = 597294, upload-time = "2025-12-04T14:50:06.847Z" }, { url = "https://files.pythonhosted.org/packages/a4/39/f1c8da50024feecd0793dbd5e08f526809b8ab5609224a2da40aad3a7641/greenlet-3.3.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e8e18ed6995e9e2c0b4ed264d2cf89260ab3ac7e13555b8032b25a74c6d18655", size = 607742, upload-time = "2025-12-04T14:57:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/77/cb/43692bcd5f7a0da6ec0ec6d58ee7cddb606d055ce94a62ac9b1aa481e969/greenlet-3.3.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c024b1e5696626890038e34f76140ed1daf858e37496d33f2af57f06189e70d7", size = 622297, upload-time = "2025-12-04T15:07:13.552Z" }, { url = "https://files.pythonhosted.org/packages/75/b0/6bde0b1011a60782108c01de5913c588cf51a839174538d266de15e4bf4d/greenlet-3.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:047ab3df20ede6a57c35c14bf5200fcf04039d50f908270d3f9a7a82064f543b", size = 609885, upload-time = "2025-12-04T14:26:02.368Z" }, { url = "https://files.pythonhosted.org/packages/49/0e/49b46ac39f931f59f987b7cd9f34bfec8ef81d2a1e6e00682f55be5de9f4/greenlet-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d9ad37fc657b1102ec880e637cccf20191581f75c64087a549e66c57e1ceb53", size = 1567424, upload-time = "2025-12-04T15:04:23.757Z" }, { url = "https://files.pythonhosted.org/packages/05/f5/49a9ac2dff7f10091935def9165c90236d8f175afb27cbed38fb1d61ab6b/greenlet-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83cd0e36932e0e7f36a64b732a6f60c2fc2df28c351bae79fbaf4f8092fe7614", size = 1636017, upload-time = "2025-12-04T14:27:29.688Z" }, @@ -2018,6 +2037,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/2f/28592176381b9ab2cafa12829ba7b472d177f3acc35d8fbcf3673d966fff/greenlet-3.3.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:a1e41a81c7e2825822f4e068c48cb2196002362619e2d70b148f20a831c00739", size = 275140, upload-time = "2025-12-04T14:23:01.282Z" }, { url = "https://files.pythonhosted.org/packages/2c/80/fbe937bf81e9fca98c981fe499e59a3f45df2a04da0baa5c2be0dca0d329/greenlet-3.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f515a47d02da4d30caaa85b69474cec77b7929b2e936ff7fb853d42f4bf8808", size = 599219, upload-time = "2025-12-04T14:50:08.309Z" }, { url = "https://files.pythonhosted.org/packages/c2/ff/7c985128f0514271b8268476af89aee6866df5eec04ac17dcfbc676213df/greenlet-3.3.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d2d9fd66bfadf230b385fdc90426fcd6eb64db54b40c495b72ac0feb5766c54", size = 610211, upload-time = "2025-12-04T14:57:43.968Z" }, + { url = "https://files.pythonhosted.org/packages/79/07/c47a82d881319ec18a4510bb30463ed6891f2ad2c1901ed5ec23d3de351f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30a6e28487a790417d036088b3bcb3f3ac7d8babaa7d0139edbaddebf3af9492", size = 624311, upload-time = "2025-12-04T15:07:14.697Z" }, { url = "https://files.pythonhosted.org/packages/fd/8e/424b8c6e78bd9837d14ff7df01a9829fc883ba2ab4ea787d4f848435f23f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:087ea5e004437321508a8d6f20efc4cfec5e3c30118e1417ea96ed1d93950527", size = 612833, upload-time = "2025-12-04T14:26:03.669Z" }, { url = "https://files.pythonhosted.org/packages/b5/ba/56699ff9b7c76ca12f1cdc27a886d0f81f2189c3455ff9f65246780f713d/greenlet-3.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ab97cf74045343f6c60a39913fa59710e4bd26a536ce7ab2397adf8b27e67c39", size = 1567256, upload-time = "2025-12-04T15:04:25.276Z" }, { url = "https://files.pythonhosted.org/packages/1e/37/f31136132967982d698c71a281a8901daf1a8fbab935dce7c0cf15f942cc/greenlet-3.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5375d2e23184629112ca1ea89a53389dddbffcf417dad40125713d88eb5f96e8", size = 1636483, upload-time = "2025-12-04T14:27:30.804Z" }, @@ -2025,6 +2045,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/7c/f0a6d0ede2c7bf092d00bc83ad5bafb7e6ec9b4aab2fbdfa6f134dc73327/greenlet-3.3.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:60c2ef0f578afb3c8d92ea07ad327f9a062547137afe91f38408f08aacab667f", size = 275671, upload-time = "2025-12-04T14:23:05.267Z" }, { url = "https://files.pythonhosted.org/packages/44/06/dac639ae1a50f5969d82d2e3dd9767d30d6dbdbab0e1a54010c8fe90263c/greenlet-3.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a5d554d0712ba1de0a6c94c640f7aeba3f85b3a6e1f2899c11c2c0428da9365", size = 646360, upload-time = "2025-12-04T14:50:10.026Z" }, { url = "https://files.pythonhosted.org/packages/e0/94/0fb76fe6c5369fba9bf98529ada6f4c3a1adf19e406a47332245ef0eb357/greenlet-3.3.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3a898b1e9c5f7307ebbde4102908e6cbfcb9ea16284a3abe15cab996bee8b9b3", size = 658160, upload-time = "2025-12-04T14:57:45.41Z" }, + { url = "https://files.pythonhosted.org/packages/93/79/d2c70cae6e823fac36c3bbc9077962105052b7ef81db2f01ec3b9bf17e2b/greenlet-3.3.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dcd2bdbd444ff340e8d6bdf54d2f206ccddbb3ccfdcd3c25bf4afaa7b8f0cf45", size = 671388, upload-time = "2025-12-04T15:07:15.789Z" }, { url = "https://files.pythonhosted.org/packages/b8/14/bab308fc2c1b5228c3224ec2bf928ce2e4d21d8046c161e44a2012b5203e/greenlet-3.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5773edda4dc00e173820722711d043799d3adb4f01731f40619e07ea2750b955", size = 660166, upload-time = "2025-12-04T14:26:05.099Z" }, { url = "https://files.pythonhosted.org/packages/4b/d2/91465d39164eaa0085177f61983d80ffe746c5a1860f009811d498e7259c/greenlet-3.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac0549373982b36d5fd5d30beb8a7a33ee541ff98d2b502714a09f1169f31b55", size = 1615193, upload-time = "2025-12-04T15:04:27.041Z" }, { url = "https://files.pythonhosted.org/packages/42/1b/83d110a37044b92423084d52d5d5a3b3a73cafb51b547e6d7366ff62eff1/greenlet-3.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d198d2d977460358c3b3a4dc844f875d1adb33817f0613f663a656f463764ccc", size = 1683653, upload-time = "2025-12-04T14:27:32.366Z" }, @@ -2032,6 +2053,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/66/bd6317bc5932accf351fc19f177ffba53712a202f9df10587da8df257c7e/greenlet-3.3.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d6ed6f85fae6cdfdb9ce04c9bf7a08d666cfcfb914e7d006f44f840b46741931", size = 282638, upload-time = "2025-12-04T14:25:20.941Z" }, { url = "https://files.pythonhosted.org/packages/30/cf/cc81cb030b40e738d6e69502ccbd0dd1bced0588e958f9e757945de24404/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9125050fcf24554e69c4cacb086b87b3b55dc395a8b3ebe6487b045b2614388", size = 651145, upload-time = "2025-12-04T14:50:11.039Z" }, { url = "https://files.pythonhosted.org/packages/9c/ea/1020037b5ecfe95ca7df8d8549959baceb8186031da83d5ecceff8b08cd2/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87e63ccfa13c0a0f6234ed0add552af24cc67dd886731f2261e46e241608bee3", size = 654236, upload-time = "2025-12-04T14:57:47.007Z" }, + { url = "https://files.pythonhosted.org/packages/69/cc/1e4bae2e45ca2fa55299f4e85854606a78ecc37fead20d69322f96000504/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2662433acbca297c9153a4023fe2161c8dcfdcc91f10433171cf7e7d94ba2221", size = 662506, upload-time = "2025-12-04T15:07:16.906Z" }, { url = "https://files.pythonhosted.org/packages/57/b9/f8025d71a6085c441a7eaff0fd928bbb275a6633773667023d19179fe815/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c6e9b9c1527a78520357de498b0e709fb9e2f49c3a513afd5a249007261911b", size = 653783, upload-time = "2025-12-04T14:26:06.225Z" }, { url = "https://files.pythonhosted.org/packages/f6/c7/876a8c7a7485d5d6b5c6821201d542ef28be645aa024cfe1145b35c120c1/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:286d093f95ec98fdd92fcb955003b8a3d054b4e2cab3e2707a5039e7b50520fd", size = 1614857, upload-time = "2025-12-04T15:04:28.484Z" }, { url = "https://files.pythonhosted.org/packages/4f/dc/041be1dff9f23dac5f48a43323cd0789cb798342011c19a248d9c9335536/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c10513330af5b8ae16f023e8ddbfb486ab355d04467c4679c5cfe4659975dd9", size = 1676034, upload-time = "2025-12-04T14:27:33.531Z" }, @@ -4259,6 +4281,9 @@ aws-nova-sonic = [ azure = [ { name = "azure-cognitiveservices-speech" }, ] +camb = [ + { name = "camb-sdk" }, +] cartesia = [ { name = "cartesia" }, { name = "websockets" }, @@ -4480,6 +4505,7 @@ requires-dist = [ { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.2.0" }, { name = "aws-sdk-sagemaker-runtime-http2", marker = "python_full_version >= '3.12' and extra == 'sagemaker'" }, { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.44.0" }, + { name = "camb-sdk", marker = "extra == 'camb'", specifier = ">=1.5.4" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.23.0" }, @@ -4487,8 +4513,8 @@ requires-dist = [ { name = "docstring-parser", specifier = "~=0.16" }, { name = "einops", marker = "extra == 'moondream'", specifier = "~=0.8.0" }, { name = "fal-client", marker = "extra == 'fal'", specifier = "~=0.5.9" }, - { name = "fastapi", marker = "extra == 'runner'", specifier = ">=0.115.6,<0.122.0" }, - { name = "fastapi", marker = "extra == 'websocket'", specifier = ">=0.115.6,<0.122.0" }, + { name = "fastapi", marker = "extra == 'runner'", specifier = ">=0.115.6,<0.128.0" }, + { name = "fastapi", marker = "extra == 'websocket'", specifier = ">=0.115.6,<0.128.0" }, { name = "faster-whisper", marker = "extra == 'whisper'", specifier = "~=1.1.1" }, { name = "google-cloud-speech", marker = "extra == 'google'", specifier = ">=2.33.0,<3" }, { name = "google-cloud-texttospeech", marker = "extra == 'google'", specifier = ">=2.31.0,<3" }, @@ -4573,7 +4599,7 @@ requires-dist = [ { name = "wait-for2", marker = "python_full_version < '3.12'", specifier = ">=0.4.1" }, { name = "websockets", marker = "extra == 'websockets-base'", specifier = ">=13.1,<16.0" }, ] -provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "gradium", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "krisp", "langchain", "livekit", "lmnt", "local", "local-smart-turn", "local-smart-turn-v3", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "neuphonic", "noisereduce", "nvidia", "openai", "rnnoise", "openpipe", "openrouter", "perplexity", "playht", "qwen", "remote-smart-turn", "rime", "riva", "runner", "sagemaker", "sambanova", "sarvam", "sentry", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] +provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "camb", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "gradium", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "krisp", "langchain", "livekit", "lmnt", "local", "local-smart-turn", "local-smart-turn-v3", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "neuphonic", "noisereduce", "nvidia", "openai", "rnnoise", "openpipe", "openrouter", "perplexity", "playht", "qwen", "remote-smart-turn", "rime", "riva", "runner", "sagemaker", "sambanova", "sarvam", "sentry", "silero", "simli", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper"] [package.metadata.requires-dev] dev = [ @@ -7558,6 +7584,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/d4/ed38dd3b1767193de971e694aa544356e63353c33a85d948166b5ff58b9e/watchfiles-1.1.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e6f39af2eab0118338902798b5aa6664f46ff66bc0280de76fca67a7f262a49", size = 457546, upload-time = "2025-10-14T15:06:13.372Z" }, ] +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + [[package]] name = "websockets" version = "13.1"