From ae00aa34a91df4b042d991ed0c4dbecfbad2ca4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Ctinalenguyen=E2=80=9D?= Date: Tue, 11 Aug 2026 14:41:05 -0400 Subject: [PATCH] update default --- README.md | 2 ++ pyproject.toml | 2 +- src/agent.py | 30 +++++++++++++++++++----------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index acce886..8c48a2b 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,10 @@ The starter project includes: - A simple voice AI assistant, ready for extension and customization - A voice AI pipeline built on [LiveKit Inference](https://docs.livekit.io/agents/models/inference), providing zero-configuration access to [models](https://docs.livekit.io/agents/models) from top labs - Uses the fast, open-weight Gemma 4 31B model, [hosted by LiveKit](https://docs.livekit.io/agents/models/llm/livekit/) and tuned for optimal performance in voice AI, as the default LLM + - Uses Fish Audio S2.1 Pro for TTS, which renders the inline delivery markup that expressive mode relies on - Supports more than 50 models from OpenAI, Cartesia, Deepgram, and other providers - Access to a wide range of other models, including [Realtime models](https://docs.livekit.io/agents/models/realtime), through extensive plugin ecosystem +- Expressive mode, enabled by default: the framework injects the TTS provider's markup guide into the LLM prompt, so the model emits inline delivery tags (emotion, pacing, non-verbal sounds) that the TTS renders and the transcript never shows - Eval suite based on the LiveKit Agents [testing & evaluation framework](https://docs.livekit.io/agents/start/testing/) - [LiveKit Turn Detector](https://docs.livekit.io/agents/logic/turns/turn-detector/), an end-of-turn model that listens to the user's audio directly, combining semantic understanding with acoustic cues for state-of-the-art accuracy across 14 languages - [Background voice cancellation](https://docs.livekit.io/transport/media/noise-cancellation/) diff --git a/pyproject.toml b/pyproject.toml index f7f4b59..87520d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "Simple voice AI assistant built with LiveKit Agents for Python" requires-python = ">=3.10, <3.15" dependencies = [ - "livekit-agents>=1.6.1", + "livekit-agents>=1.6.9", "livekit-plugins-ai-coustics~=0.2", "python-dotenv", ] diff --git a/src/agent.py b/src/agent.py index 2fd5e3d..17e6a91 100644 --- a/src/agent.py +++ b/src/agent.py @@ -99,27 +99,35 @@ async def my_agent(ctx: JobContext): "room": ctx.room.name, } - # Set up a voice AI pipeline using OpenAI, Cartesia, Deepgram, and the LiveKit turn detector + # Set up a voice AI pipeline using AssemblyAI, Fish Audio, and the LiveKit turn detector session = AgentSession( # Speech-to-text (STT) is your agent's ears, turning the user's speech into text that the LLM can understand # See all available models at https://docs.livekit.io/agents/models/stt/ - stt=inference.STT(model="deepgram/nova-3", language="multi"), + stt=inference.STT(model="assemblyai/universal-3-5-pro", language="en"), # Text-to-speech (TTS) is your agent's voice, turning the LLM's text into speech that the user can hear # See all available models as well as voice selections at https://docs.livekit.io/agents/models/tts/ tts=inference.TTS( - model="cartesia/sonic-3", voice="9626c31c-bec5-4cca-baa8-f8ba9e84c8bc" + model="fishaudio/s2.1-pro", voice="fa4c9eb3dccc4806b382b40d61c6b10a" ), - # The LiveKit turn detector determines when the user is done speaking and the agent should respond. - # TurnDetector is an end-of-turn model that listens to the user's audio directly, combining - # semantic understanding with acoustic cues (intonation, pitch, rhythm) for state-of-the-art accuracy. - # AgentSession supplies the required VAD automatically. - # See more at https://docs.livekit.io/agents/build/turns turn_handling=TurnHandlingOptions( + # The LiveKit turn detector determines when the user is done speaking and the agent should respond. + # TurnDetector is an end-of-turn model that listens to the user's audio directly, combining + # semantic understanding with acoustic cues (intonation, pitch, rhythm) for state-of-the-art accuracy. + # AgentSession supplies the required VAD automatically. + # See more at https://docs.livekit.io/agents/build/turns turn_detection=inference.TurnDetector(), + # Adaptive interruptions use the turn detector to tell a real interruption from a + # backchannel like "mhm" or "right", so the agent keeps talking through the latter. + interruption={"mode": "adaptive"}, + # allow the LLM to generate a response while waiting for the end of turn + # See more at https://docs.livekit.io/agents/build/audio/#preemptive-generation + preemptive_generation={"enabled": True}, ), - # allow the LLM to generate a response while waiting for the end of turn - # See more at https://docs.livekit.io/agents/build/audio/#preemptive-generation - preemptive_generation=True, + # Expressive mode injects the TTS provider's markup guide into the LLM prompt, so the model + # emits inline delivery tags (emotion, pacing, non-verbal sounds) that the TTS renders and + # the transcript never shows. Requires a TTS model that supports markup, such as the Fish + # Audio model above. + expressive=True, ) # Start the session, which initializes the voice pipeline and warms up the models