diff --git a/.cargo/config.toml b/.cargo/config.toml index d42c757d31..594c7b357f 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -9,3 +9,10 @@ debug = "line-tables-only" # cmake_minimum_required < 3.5; audiopus_sys's vendored opus declares 3.1. # Same workaround CI uses. A value already set in the environment wins. CMAKE_POLICY_VERSION_MINIMUM = "3.5" + +# MSVC otherwise reports the legacy C++98 value for `__cplusplus`, even when +# cc-rs selects C++14. sstretch uses that macro to gate its C++11 +# `std::make_unique` polyfill, which collides with the MSVC standard library. +# Match sherpa-onnx's static MSVC runtime so both native audio libraries can +# link into the same Tauri binary without LNK2038 RuntimeLibrary mismatches. +CXXFLAGS_x86_64_pc_windows_msvc = "/Zc:__cplusplus /MT" diff --git a/Cargo.lock b/Cargo.lock index 3b60dc4579..5a57e203d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1262,6 +1262,14 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "serde_json", + "sherpa-onnx", +] + [[package]] name = "buzz-workflow" version = "0.1.0" @@ -1329,6 +1337,26 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "castaway" version = "0.2.4" @@ -8049,6 +8077,28 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "sherpa-onnx" +version = "1.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b142d3f255cb4e4b7808ea25869db6f5714e0a3550da355234483b4db552055" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc951af03dc0653c0622158ca8a585a6f2bc43b7b06048cf0e5b5020005c227" +dependencies = [ + "bzip2", + "tar", + "ureq", +] + [[package]] name = "shlex" version = "1.3.0" @@ -9539,6 +9589,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.8" diff --git a/Cargo.toml b/Cargo.toml index 3ac7ee4cce..3268cfaf8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "crates/buzz-pair-relay", "crates/buzz-relay-mesh", "crates/buzz-dev-mcp", + "crates/buzz-voice", "examples/countdown-bot", ] exclude = ["desktop/src-tauri"] diff --git a/Justfile b/Justfile index bcef8983bc..e5601607ba 100644 --- a/Justfile +++ b/Justfile @@ -276,6 +276,7 @@ test-unit: #!/usr/bin/env bash if command -v cargo-nextest &>/dev/null; then cargo nextest run -p buzz-core -p buzz-auth --lib + cargo nextest run -p buzz-voice --lib # buzz-db migrator/lint tests: pure SQL-parsing unit tests (no infra). # They guard the embedded-migrator invariant (exactly the consolidated # 0001; cutover/backfill stays an operator script, not startup state) diff --git a/crates/buzz-voice/Cargo.toml b/crates/buzz-voice/Cargo.toml new file mode 100644 index 0000000000..f577e3d1a6 --- /dev/null +++ b/crates/buzz-voice/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "buzz-voice" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Reusable local voice primitives for Buzz" + +[dependencies] +serde_json = { workspace = true } +sherpa-onnx = "1.12" diff --git a/crates/buzz-voice/src/lib.rs b/crates/buzz-voice/src/lib.rs new file mode 100644 index 0000000000..e2ce3f255b --- /dev/null +++ b/crates/buzz-voice/src/lib.rs @@ -0,0 +1,3 @@ +//! Reusable local voice primitives for Buzz. + +pub mod pocket; diff --git a/crates/buzz-voice/src/pocket.rs b/crates/buzz-voice/src/pocket.rs new file mode 100644 index 0000000000..ae235b93eb --- /dev/null +++ b/crates/buzz-voice/src/pocket.rs @@ -0,0 +1,640 @@ +//! Pocket TTS engine wrapper around sherpa-onnx's `OfflineTts`. +//! +//! Pocket TTS is a small (~473 MB fp32 ONNX) zero-shot voice-cloning TTS +//! model from Kyutai that runs quickly on CPU via sherpa-onnx. +//! +//! Buzz uses full-precision fp32 sessions because a direct same-runtime A/B +//! (k2-fsa/sherpa-onnx#3172) found the ~189 MB int8 ONNX export audibly +//! degraded output quality. +//! +//! ## Attribution +//! +//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. +//! arXiv:2509.06926. Original repository: . +//! Licensed CC-BY-4.0. +//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. +//! - **ONNX export**: KevinAHM — +//! . CC-BY-4.0. +//! - **sherpa-onnx repackage**: csukuangfj / k2-fsa — +//! . +//! Repackages KevinAHM's export with the file layout sherpa-onnx's +//! `OfflineTtsPocketModelConfig` expects. CC-BY-4.0. +//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary +//! (f, conversation)" preset from the Kyutai TTS demo +//! (), which maps to `vctk/p333_023_enhanced.wav` +//! in . CC-BY-4.0, base recording +//! from the VCTK corpus, enhanced by ai-coustics. +//! +//! Buzz ships these files unmodified; see the on-disk `MODEL_LICENSE.txt` +//! sidecar written by `huddle::models` during install for the canonical +//! CC-BY-4.0 §3(a)(1) attribution block. +//! +//! ## Engine-module contract (see `huddle::tts`) +//! +//! `pocket.rs` exposes a fixed surface used by `tts.rs`. Mirroring this +//! contract is what lets the TTS pipeline stay engine-agnostic: +//! +//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. +//! - `DEFAULT_VOICE: &str` — default voice name (without extension). +//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. +//! - `load_text_to_speech(model_dir)` → `Result` +//! - `load_voice_style(path)` → `Result` +//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` +//! → `Result, String>` +//! +//! `lang` and `steps` are accepted for API compatibility with the previous +//! Kokoro engine but are unused — Pocket TTS does its own language ID from +//! the input text and is not a diffusion model (consistency LM, one step). +//! There is no speed knob: sherpa-onnx's `GenerationConfig.speed` is only +//! read by some model families (vits), never by the Pocket impl +//! (`offline-tts-pocket-impl.h` — zero references), and upstream pocket-tts +//! has no speed parameter either. + +use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; + +use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; + +// ── Engine-module contract: public consts ───────────────────────────────────── + +/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, +/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. +pub const SAMPLE_RATE: u32 = 24_000; + +/// Name (without extension) of the bundled reference voice. The model directory +/// is expected to contain `.` after install. +pub const DEFAULT_VOICE: &str = "reference_sample"; + +/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the +/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, +/// not a precomputed embedding, so the extension change is honest. +pub const VOICE_FILE_EXT: &str = "wav"; + +// ── Tuning ──────────────────────────────────────────────────────────────────── + +/// Single-threaded ONNX execution for predictable CPU contention with the STT +/// pipeline. Matches `STT_NUM_THREADS` in `stt.rs`; raise only if a benchmark +/// argues for it. +const TTS_NUM_THREADS: i32 = 1; + +/// LRU cache size for cloned voice embeddings inside the sherpa-onnx engine. +/// We bind to one voice per pipeline today, but the upstream example uses 16 +/// and the cost is negligible — keep room for future multi-voice support. +const VOICE_EMBEDDING_CACHE_CAPACITY: i32 = 16; + +/// Pocket TTS is a consistency-based LM. Generation quality saturates at one +/// denoising step — the upstream `GenerationConfig` default of 5 multiplies +/// synthesis time by ~5× with no audible benefit on this model. +const SYNTH_NUM_STEPS: i32 = 1; + +/// Leave the generated audio's silences untouched (1.0 is the identity). +/// +/// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding +/// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and +/// multiplies its length by this factor. The reference Pocket TTS pipeline +/// preserves natural clause breaks, breaths, and punctuation pauses, so the +/// identity scale keeps those interior silences intact. +const SYNTH_SILENCE_SCALE: f32 = 1.0; + +/// sherpa-onnx upstream default for `max_frames` (LM steps), in +/// `offline-tts-pocket-impl.h:Generate`. 500 steps ≈ 40 s of audio at the +/// Mimi 12.5 Hz frame rate. Referenced only by the regression test below; +/// production code path never raises (or even reads) this value — we just +/// leave sherpa-onnx's own default in place by not setting the override. +#[cfg(test)] +const SHERPA_ONNX_MAX_FRAMES_DEFAULT: i32 = 500; + +/// Tight `max_frames` we ask for on short prompts to bound the +/// original "monster breathing" runaway. 100 LM steps ≈ 8 s of audio — +/// roomy for any one-to-four-word utterance the user is likely to elicit +/// while still well short of the 40 s upstream default. Chosen with slack so +/// we never *truncate* a legitimate short reply. +const SHORT_PROMPT_MAX_FRAMES: i32 = 100; + +/// Word-count threshold (inclusive) below which we cap `max_frames` tighter +/// than the upstream default. Above this threshold we leave sherpa-onnx's +/// generation limits in place because dropping `frames_after_eos` below the +/// upstream default of 3 can clip the leading audio of multi-clause sentences. +const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; + +/// sherpa-onnx's documented `frames_after_eos` default. We deliberately do +/// *not* override this knob because values below the upstream default of 3 can +/// clip the leading audio of multi-clause sentences. The constant exists only +/// for the invariant test below. Source: `offline-tts-pocket-impl.h:Generate`. +#[cfg(test)] +const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; + +// ── ONNX file names (five Pocket TTS sessions plus two JSON tables) ─────────── + +const FILE_LM_MAIN: &str = "lm_main.onnx"; +const FILE_LM_FLOW: &str = "lm_flow.onnx"; +const FILE_ENCODER: &str = "encoder.onnx"; +const FILE_DECODER: &str = "decoder.onnx"; +const FILE_TEXT_COND: &str = "text_conditioner.onnx"; +const FILE_VOCAB: &str = "vocab.json"; +const FILE_TOKEN_SCORES: &str = "token_scores.json"; + +// ── Voice style ─────────────────────────────────────────────────────────────── + +/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. +/// +/// Pocket TTS takes a reference waveform per generation call (not a +/// precomputed style embedding), so we keep the samples in memory and clone +/// the small `Vec` into each `GenerationConfig` rather than re-reading the +/// WAV from disk on every sentence. +#[derive(Debug, Clone)] +pub struct VoiceStyle { + samples: Vec, + sample_rate: i32, +} + +/// Load a reference voice WAV from disk. +/// +/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS +/// resamples internally using `reference_sample_rate`. The bundled +/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. +pub fn load_voice_style(path: &Path) -> Result { + let path_str = path + .to_str() + .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; + let wave = Wave::read(path_str) + .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; + let samples = wave.samples().to_vec(); + if samples.is_empty() { + return Err(format!("voice WAV is empty: {}", path.display())); + } + Ok(VoiceStyle { + samples, + sample_rate: wave.sample_rate(), + }) +} + +// ── Engine ──────────────────────────────────────────────────────────────────── + +/// Pocket TTS engine handle. Cheap to construct (one `OfflineTts::create` +/// call). Owned by the TTS worker thread for the lifetime of a huddle session. +/// +/// `OfflineTts` does not implement `Debug`, so we don't derive it here — the +/// pipeline only needs to move the engine into the worker thread and call +/// `synth_chunk` on it, never to print it. +pub struct PocketTts { + inner: OfflineTts, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum SynthesisOutcome { + Complete(Vec), + Interrupted, +} + +/// Build the Pocket TTS engine from the model directory installed by +/// `huddle::models`. Returns `Err` if any expected ONNX or JSON file is +/// missing — readiness is normally enforced by `is_tts_ready` upstream, but +/// the check is repeated here so a manually-modified model dir produces a +/// clear error string instead of an opaque sherpa-onnx `None`. +pub fn load_text_to_speech(model_dir: &str) -> Result { + let dir = PathBuf::from(model_dir); + for name in [ + FILE_LM_MAIN, + FILE_LM_FLOW, + FILE_ENCODER, + FILE_DECODER, + FILE_TEXT_COND, + FILE_VOCAB, + FILE_TOKEN_SCORES, + ] { + let p = dir.join(name); + if !p.is_file() { + return Err(format!("missing Pocket TTS file: {}", p.display())); + } + } + + let to_str = |name: &str| -> String { dir.join(name).to_string_lossy().into_owned() }; + + // Build the config by mutating defaults — mirrors `stt.rs` and stays + // resilient if sherpa-onnx adds unrelated model-family fields. + let mut cfg = OfflineTtsConfig::default(); + cfg.model.pocket.lm_main = Some(to_str(FILE_LM_MAIN)); + cfg.model.pocket.lm_flow = Some(to_str(FILE_LM_FLOW)); + cfg.model.pocket.encoder = Some(to_str(FILE_ENCODER)); + cfg.model.pocket.decoder = Some(to_str(FILE_DECODER)); + cfg.model.pocket.text_conditioner = Some(to_str(FILE_TEXT_COND)); + cfg.model.pocket.vocab_json = Some(to_str(FILE_VOCAB)); + cfg.model.pocket.token_scores_json = Some(to_str(FILE_TOKEN_SCORES)); + cfg.model.pocket.voice_embedding_cache_capacity = VOICE_EMBEDDING_CACHE_CAPACITY; + cfg.model.num_threads = TTS_NUM_THREADS; + // Explicit — defaults are not part of the API contract, and noisy debug + // logging in release builds would be expensive on every synthesized chunk. + cfg.model.debug = false; + + let inner = OfflineTts::create(&cfg) + .ok_or_else(|| "OfflineTts::create returned None for Pocket TTS".to_string())?; + Ok(PocketTts { inner }) +} + +// ── Prompt preparation ──────────────────────────────────────────────────────── + +/// Result of [`prepare_pocket_prompt`]: a synthesizer-ready prompt plus the +/// per-call generation overrides derived from the original text. +/// +/// `None` for either override means "leave sherpa-onnx's documented default +/// in place". The pipeline only sets `max_frames` for short prompts, bounding +/// runaway generation without disturbing the rest of the LM sampling envelope. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct PreparedPrompt { + /// Text to hand to `OfflineTts::generate_with_config`. Capitalized, + /// punctuation-terminated, and prefixed with the production `. ` cold-start + /// mitigation. + pub text: String, + /// Value to pass via `GenerationConfig.extra["max_frames"]`, or `None` to + /// keep the upstream default of 500 LM steps. We only override on short + /// prompts where we have a tight expectation on output length. + pub max_frames: Option, +} + +/// Mirror of the *text-preparation* half of upstream +/// `pocket_tts.models.tts_model.prepare_text_prompt`. Sherpa-onnx's C++ +/// Pocket TTS impl does not run these preparation steps, so short / +/// unpunctuated / lowercase inputs can trigger up to 40 s of runaway +/// generation when the EOS logit never crosses its threshold. +/// +/// We mirror the upstream Python recipe by: +/// +/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but +/// cheap to re-check after sentence splitting). +/// 2. Capitalize the first letter. +/// 3. Append `.` if the text doesn't end in punctuation. +/// 4. If fewer than five words, return a tight +/// [`SHORT_PROMPT_MAX_FRAMES`] cap so the LM can't run away if EOS still +/// doesn't fire. +/// +/// Buzz also prepends `". "` to every prompt. This sacrificial punctuation +/// lets the autoregressive model settle before it renders the first real word. +/// +/// We do **not** override `frames_after_eos` — sherpa-onnx's default of 3 +/// is what we want. An earlier version set it to 1 on long inputs, which +/// clipped the leading audio of multi-clause sentences ("first 'yep' is +/// just static" regression). Tests `prepare_prompt_never_lowers_frames_…` +/// lock this in. +/// +/// Returns `None` only if the input is empty after trimming — caller should +/// skip synthesis in that case. +pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { + let trimmed = input.trim(); + if trimmed.is_empty() { + return None; + } + + // Collapse stray double-spaces / embedded newlines that may slip past + // `preprocess_for_tts` when sentences are spliced back together. + let mut cleaned = String::with_capacity(trimmed.len()); + let mut last_was_space = false; + for ch in trimmed.chars() { + let is_ws = ch.is_whitespace(); + if is_ws { + if !last_was_space { + cleaned.push(' '); + } + last_was_space = true; + } else { + cleaned.push(ch); + last_was_space = false; + } + } + + // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). + let first = cleaned.chars().next().expect("cleaned non-empty above"); + if first.is_lowercase() { + let upper: String = first.to_uppercase().collect(); + let mut iter = cleaned.chars(); + iter.next(); + cleaned = upper + iter.as_str(); + } + + // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. + // The upstream Python only checks `isalnum` → period, but for our agent + // text we already may end in `!` `?` `.` etc. — treat any of those as OK. + let last = cleaned + .chars() + .next_back() + .expect("cleaned non-empty above"); + if !matches!(last, '.' | '!' | '?' | ';' | ':' | ',') { + cleaned.push('.'); + } + + // Count words before adding the cold-start prefix. + let word_count = cleaned.split_whitespace().count(); + + let max_frames = if word_count <= SHORT_PROMPT_WORD_THRESHOLD { + Some(SHORT_PROMPT_MAX_FRAMES) + } else { + // For everything ≥5 words, keep the upstream generation limits. + None + }; + + let mut final_text = String::with_capacity(cleaned.len() + 2); + final_text.push_str(". "); + final_text.push_str(&cleaned); + + Some(PreparedPrompt { + text: final_text, + max_frames, + }) +} + +/// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. +/// +/// Centralised so the invariant test below can assert that we **never** emit a +/// `frames_after_eos` override. Leaving the key unset preserves sherpa-onnx's +/// upstream default of 3. +fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { + prepared.max_frames.map(|mf| { + let mut h: HashMap = HashMap::with_capacity(1); + h.insert("max_frames".to_string(), serde_json::Value::from(mf)); + h + }) +} + +impl PocketTts { + /// Synthesise `text` with the given reference voice. + /// + /// `_lang` and `_steps` are accepted for API compatibility with the + /// previous Kokoro engine. Pocket TTS infers language from the input text + /// directly and is a one-step consistency model. Returns an empty buffer + /// for whitespace-only input. + pub fn synth_chunk( + &self, + text: &str, + lang: &str, + style: &VoiceStyle, + steps: usize, + ) -> Result, String> { + match self.synth_chunk_interruptible(text, lang, style, steps, || false)? { + SynthesisOutcome::Complete(samples) => Ok(samples), + SynthesisOutcome::Interrupted => Ok(Vec::new()), + } + } + + /// Synthesise one chunk while allowing the caller to stop in-flight model + /// generation. Interrupted output is discarded, so a partial waveform can + /// never be queued after a barge-in, voice change, or shutdown. + pub fn synth_chunk_interruptible( + &self, + text: &str, + _lang: &str, + style: &VoiceStyle, + _steps: usize, + is_interrupted: F, + ) -> Result + where + F: Fn() -> bool + Send + Sync + 'static, + { + // Mirror upstream pocket-tts prompt prep — without this short or + // unpunctuated inputs can cause the LM's EOS logit to never trip, + // producing up to 40 s of "monster breathing" garbage on the first + // utterance. See `prepare_pocket_prompt` for the full recipe. + let prepared = match prepare_pocket_prompt(text) { + Some(p) => p, + None => return Ok(SynthesisOutcome::Complete(Vec::new())), + }; + let is_interrupted = Arc::new(is_interrupted); + if is_interrupted() { + return Ok(SynthesisOutcome::Interrupted); + } + + // Per-call generation hints sherpa-onnx forwards to + // `offline-tts-pocket-impl.h`. We only override `max_frames`, and + // only for short padded prompts where we have a tight expectation + // on output length — that bounds the original runaway without + // disturbing the rest of the LM sampling envelope. See + // `prepare_pocket_prompt` docs for the regression history. + let extra = build_generation_extra(&prepared); + + let cfg = GenerationConfig { + num_steps: SYNTH_NUM_STEPS, + silence_scale: SYNTH_SILENCE_SCALE, + reference_audio: Some(style.samples.clone()), + reference_sample_rate: style.sample_rate, + extra, + // `speed` stays at its default: the Pocket impl never reads it + // (see the engine-contract note in the module docs). + ..Default::default() + }; + + let callback_interrupted = Arc::new(AtomicBool::new(false)); + let callback_flag = Arc::clone(&callback_interrupted); + let callback_predicate = Arc::clone(&is_interrupted); + let audio = self + .inner + .generate_with_config( + &prepared.text, + &cfg, + Some(move |_samples: &[f32], _progress: f32| { + let interrupted = callback_predicate(); + if interrupted { + callback_flag.store(true, Ordering::Release); + } + !interrupted + }), + ) + .ok_or_else(|| { + format!( + "Pocket TTS synthesis failed for text ({} chars)", + prepared.text.len() + ) + })?; + + let sample_rate = audio.sample_rate(); + if sample_rate != SAMPLE_RATE as i32 { + eprintln!( + "buzz-desktop: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ + (expected {SAMPLE_RATE}Hz); playback speed may be wrong" + ); + } + + if callback_interrupted.load(Ordering::Acquire) || is_interrupted() { + Ok(SynthesisOutcome::Interrupted) + } else { + Ok(SynthesisOutcome::Complete(audio.samples().to_vec())) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── prepare_pocket_prompt ──────────────────────────────────────────────── + + #[test] + fn prepare_prompt_returns_none_for_empty_input() { + assert!(prepare_pocket_prompt("").is_none()); + assert!(prepare_pocket_prompt(" ").is_none()); + assert!(prepare_pocket_prompt("\n\t ").is_none()); + } + + /// Production's cold-start prefix. + fn prompt_prefix() -> &'static str { + ". " + } + + #[test] + fn prepare_prompt_prefixes_and_capitalizes_one_word() { + // A bare lowercase word is prefixed, capitalized, terminated, and + // capped tightly enough to bound runaway generation. + let out = prepare_pocket_prompt("yep").expect("non-empty"); + assert_eq!(out.text, format!("{}Yep.", prompt_prefix())); + assert_eq!(out.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); + const { + assert!( + SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT, + "short cap must be tighter than the upstream default" + ); + } + } + + #[test] + fn prepare_prompt_preserves_existing_punctuation() { + let out = prepare_pocket_prompt("yes!").expect("non-empty"); + assert_eq!(out.text, format!("{}Yes!", prompt_prefix())); // exclamation kept + let out = prepare_pocket_prompt("really?").expect("non-empty"); + assert_eq!(out.text, format!("{}Really?", prompt_prefix())); + } + + #[test] + fn prepare_prompt_threshold_is_inclusive_at_four_words() { + // Both prompts are prefixed. Only the 4-word prompt gets a tight + // max_frames cap; the 5-word prompt keeps upstream generation limits. + let four = prepare_pocket_prompt("one two three four").expect("non-empty"); + assert_eq!( + four.text, + format!("{}One two three four.", prompt_prefix()), + "four-word input should get exactly the production prefix" + ); + assert_eq!(four.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); + + let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); + assert_eq!( + five.text, + format!("{}One two three four five.", prompt_prefix()), + "five-word input should also get the cold-start prefix" + ); + assert_eq!( + five.max_frames, None, + "long inputs must leave sherpa-onnx's max_frames default in place" + ); + } + + #[test] + fn prepare_prompt_prefixes_long_text_without_capping_it() { + let long = "This is a longer sentence that the model should handle just fine."; + let out = prepare_pocket_prompt(long).expect("non-empty"); + assert_eq!(out.text, format!("{}{}", prompt_prefix(), long)); + assert_eq!(out.max_frames, None); + assert!(out.text.ends_with('.')); + } + + #[test] + fn prepare_prompt_collapses_whitespace() { + let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); + // 3 words → short → prefixed. Interior whitespace collapsed. + assert_eq!(out.text, format!("{}Hello world friend.", prompt_prefix())); + } + + #[test] + fn prepare_prompt_does_not_double_capitalize_already_uppercase() { + let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); + assert_eq!(out.text, format!("{}HELLO there.", prompt_prefix())); + } + + #[test] + fn prepare_prompt_handles_non_ascii_first_letter() { + // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce + // mojibake. + let out = prepare_pocket_prompt("дa").expect("non-empty"); + assert!(out.text.contains("Дa.")); + } + + // ── build_generation_extra ─────────────────────────────────────────────── + // + // Short prompts override only `max_frames`; long prompts emit no extras. + // Every other knob remains at sherpa-onnx's documented default, notably + // `frames_after_eos = 3`. + + #[test] + fn build_extra_short_prompt_sets_only_max_frames() { + let prepared = prepare_pocket_prompt("yep").expect("non-empty"); + let extra = build_generation_extra(&prepared).expect("short prompts get extra"); + // Exactly one key — `max_frames` — and nothing else. + assert_eq!(extra.len(), 1, "extra has unexpected keys: {extra:?}"); + assert_eq!( + extra.get("max_frames"), + Some(&serde_json::Value::from(SHORT_PROMPT_MAX_FRAMES)) + ); + assert!( + !extra.contains_key("frames_after_eos"), + "frames_after_eos must never be set — upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT} is what we want" + ); + } + + #[test] + fn build_extra_long_prompt_is_none() { + // ≥5 words: no extras, so the upstream LM defaults remain authoritative. + let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); + assert_eq!( + build_generation_extra(&prepared), + None, + "long prompts must not override any LM knob" + ); + } + + #[test] + fn build_extra_never_lowers_frames_after_eos_for_any_word_count() { + // Sweep a range of prompt lengths and assert the `extra` map (when + // present) never carries a `frames_after_eos` override that's lower + // than the upstream sherpa-onnx default. Implemented as a structural + // check — we just never set the key — but worth a property test in + // case someone reintroduces the override in the future. + let prompts: &[&str] = &[ + "hi", + "hi there", + "yes please", + "one two three four", + "one two three four five", + "a slightly longer reply, hopefully fine", + "This is a multi-clause sentence. It has two parts.", + "really really really really really long prompt with lots of words just to be sure", + ]; + for &p in prompts { + let prepared = prepare_pocket_prompt(p).expect("non-empty"); + if let Some(extra) = build_generation_extra(&prepared) { + if let Some(v) = extra.get("frames_after_eos") { + let n = v.as_i64().expect("frames_after_eos should be int"); + assert!( + n >= SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT as i64, + "prompt {p:?} set frames_after_eos={n}, below upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT}" + ); + } + } + } + } + + #[test] + fn short_prompt_max_frames_is_below_upstream_default() { + // Sanity: the override only ever *lowers* the cap, never raises it. + const { + assert!(SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT); + } + // …and is still large enough for a one-to-four-word reply. At Mimi's + // 12.5 Hz frame rate, 100 frames = 8 s, which is roomy. + const { + assert!(SHORT_PROMPT_MAX_FRAMES >= 50, "would risk truncation"); + } + } +} diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index 40dd5dd1b1..bbdccd6de3 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -48,6 +48,7 @@ export default defineConfig({ "**/activity-scope-label-screenshots.spec.ts", "**/welcome-agent-modal-screenshots.spec.ts", "**/local-archive-screenshots.spec.ts", + "**/voice-settings.spec.ts", "**/agent-readiness-screenshots.spec.ts", "**/agent-error-state-screenshots.spec.ts", "**/edit-agent.spec.ts", diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index 5f6f48eb14..7abf14b450 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -485,7 +485,10 @@ const overrides = new Map([ // +4 (1081 -> 1085): mesh recovery keeps one app-scoped state object beside // the embedded runtime and coordinator. Probe/re-arm logic lives in // mesh_llm/recovery.rs rather than growing AppState or command modules. - ["src-tauri/src/app_state.rs", 1085], + // Installation-global TTS settings, load diagnostics, and transition + // serialization live beside the huddle state they govern. + // Voice behavior remains split across the huddle modules. + ["src-tauri/src/app_state.rs", 1091], // multi-slot splitting + no-op suppression (#1309): the ReadStateManager // class grew from ~700 lines to ~1019 with the addition of // splitContextsIntoBudgetedSlots (pure fn + 5 tests), publishSplitSlots, diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 7dcf615c00..923e614312 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -991,6 +991,7 @@ dependencies = [ name = "buzz-core" version = "0.1.0" dependencies = [ + "base64 0.22.1", "chrono", "hex", "hmac 0.13.0", @@ -1022,6 +1023,7 @@ dependencies = [ "buzz-media", "buzz-persona", "buzz-sdk", + "buzz-voice", "bytes", "bzip2 0.6.1", "chrono", @@ -1062,6 +1064,7 @@ dependencies = [ "serde_yaml", "sha2 0.11.0", "sherpa-onnx", + "ssstretch", "strip-ansi-escapes", "tar", "tauri", @@ -1142,6 +1145,14 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "serde_json", + "sherpa-onnx", +] + [[package]] name = "by_address" version = "1.2.1" @@ -1489,6 +1500,17 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -2050,6 +2072,68 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "cxx" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe442a792c7c736eea18b32a7f8a3b63cf8aafabda6760042dc2fdeda456291" +dependencies = [ + "cc", + "cxx-build", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash 0.2.0", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3184a94384c663718698311a78a51ac00c484c10b4eeac06fb0a068c5f64fa2" +dependencies = [ + "cc", + "codespan-reporting", + "indexmap 2.14.0", + "proc-macro2", + "quote", + "scratch", + "syn 3.0.3", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0148d8fd1199329ddf1d157a5e134e51ceff37c6a7ddd38615c399d81cb05d8d" +dependencies = [ + "clap", + "codespan-reporting", + "indexmap 2.14.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52850339faed2eaadd24e286dc1d8268cc6f8a7bd9524d713adc9099566b4c89" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c77c856545d886c9bd5215409ebb63b925e262135248b50c79e5a5f194ee47c" +dependencies = [ + "indexmap 2.14.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "darling" version = "0.20.11" @@ -4694,6 +4778,15 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "link-cplusplus" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" +dependencies = [ + "cc", +] + [[package]] name = "link-section" version = "0.19.0" @@ -8507,6 +8600,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" + [[package]] name = "scrypt" version = "0.11.0" @@ -9264,6 +9363,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ssstretch" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ee31a0a494b76c8d3047aac804c5f4eb4b6e96c75414e3043b2212301bb8c6" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -9544,6 +9653,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -10164,6 +10284,15 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "termina" version = "0.3.3" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index d689544688..8e867dcb80 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -89,6 +89,7 @@ buzz_core_pkg = { package = "buzz-core", path = "../../crates/buzz-core" } buzz_persona_pkg = { package = "buzz-persona", path = "../../crates/buzz-persona" } buzz_sdk_pkg = { package = "buzz-sdk", path = "../../crates/buzz-sdk" } buzz_agent_pkg = { package = "buzz-agent", path = "../../crates/buzz-agent" } +buzz_voice_pkg = { package = "buzz-voice", path = "../../crates/buzz-voice" } iroh = { version = "1.0.2", optional = true } mesh-llm-sdk = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-sdk", default-features = false, features = ["client", "serving"], optional = true } mesh-llm-host-runtime = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-host-runtime", default-features = false, features = ["dynamic-native-runtime"], optional = true } @@ -120,6 +121,7 @@ regex = "1" rusqlite = { version = "0.37", features = ["bundled"] } axum = "0.8" rodio = "0.22" +ssstretch = "0.1.0" earshot = "1.0" rubato = "3.0" audioadapter-buffers = "3.0" diff --git a/desktop/src-tauri/examples/pocket_quality_ab.rs b/desktop/src-tauri/examples/pocket_quality_ab.rs index 0c31f1c910..95f2e4815f 100644 --- a/desktop/src-tauri/examples/pocket_quality_ab.rs +++ b/desktop/src-tauri/examples/pocket_quality_ab.rs @@ -19,7 +19,7 @@ // standalone corpus generator deliberately does not call. #![allow(dead_code)] -#[path = "../src/huddle/pocket.rs"] +#[path = "../../../crates/buzz-voice/src/pocket.rs"] mod production_pocket; #[path = "../src/huddle/preprocessing.rs"] mod production_preprocessing; diff --git a/desktop/src-tauri/examples/tts_speed_probe.rs b/desktop/src-tauri/examples/tts_speed_probe.rs new file mode 100644 index 0000000000..6a9559706a --- /dev/null +++ b/desktop/src-tauri/examples/tts_speed_probe.rs @@ -0,0 +1,41 @@ +use std::time::Instant; + +#[path = "../src/huddle/playback_speed_dsp.rs"] +mod playback_speed_dsp; + +const SAMPLE_RATE: u32 = 24_000; +const SENTENCE_LEAD_IN_SAMPLES: usize = 480; +const TONE_SECONDS: f64 = 10.0; + +fn main() { + let tone: Vec = (0..SAMPLE_RATE * TONE_SECONDS as u32) + .map(|index| { + (2.0 * std::f32::consts::PI * 220.0 * index as f32 / SAMPLE_RATE as f32).sin() * 0.25 + }) + .collect(); + let mut input = Vec::with_capacity(SENTENCE_LEAD_IN_SAMPLES + tone.len()); + input.resize(SENTENCE_LEAD_IN_SAMPLES, 0.0); + input.extend_from_slice(&tone); + + let compensated_lookahead = playback_speed_dsp::compensated_output_latency_samples(SAMPLE_RATE); + let compensated_lookahead_ms = compensated_lookahead as f64 * 1_000.0 / SAMPLE_RATE as f64; + + for speed in [0.75_f32, 1.25, 1.5] { + let started = Instant::now(); + let output = playback_speed_dsp::process_complete_chunk_preserving_lead_in( + &input, + SENTENCE_LEAD_IN_SAMPLES, + speed, + SAMPLE_RATE, + ) + .expect("process production-shaped buffer"); + let elapsed = started.elapsed(); + println!( + "{speed:.2}x: processing={:.2}ms, {:.3}% realtime, output={} samples, compensated \ + lookahead={compensated_lookahead_ms:.1}ms", + elapsed.as_secs_f64() * 1_000.0, + elapsed.as_secs_f64() / TONE_SECONDS * 100.0, + output.len(), + ); + } +} diff --git a/desktop/src-tauri/resources/pocket-voices/NOTICE.md b/desktop/src-tauri/resources/pocket-voices/NOTICE.md new file mode 100644 index 0000000000..ecbee2691d --- /dev/null +++ b/desktop/src-tauri/resources/pocket-voices/NOTICE.md @@ -0,0 +1,25 @@ +# Marius Pocket TTS reference voice + +`marius.wav` is an audio-identical copy of Kyutai's +`voice-donations/Selfie.wav`; only the filename changed. + +- Display name: Marius +- Source: https://huggingface.co/kyutai/tts-voices/blob/main/voice-donations/Selfie.wav +- Upstream revision: `323332d33f997de8394f24a193e1a76df720e01a` +- SHA-256: `076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16` +- Format: WAV PCM signed 16-bit little-endian, mono, 24 kHz, 10 seconds +- License: CC0 1.0 Universal + (https://creativecommons.org/publicdomain/zero/1.0/) +- Repository provenance: + https://huggingface.co/kyutai/tts-voices/blob/main/README.md +- Donation terms: + https://kyutai.org/next/legal/Terms%20of%20Use%20-%20Unmute%20Voice%20Donation%20Project%20v1.pdf + +The adult contributor submitted their own natural voice through Kyutai's +Unmute Voice Donation Project. Its terms expressly contemplate public reuse, +text-to-speech and voice-replica use, and synthetic speech generated in the +contributed voice. + +Neither Kyutai nor the contributor endorses Buzz. Buzz does not identify or +attempt to identify the contributor, and synthetic output must not be presented +as a genuine recording by them. diff --git a/desktop/src-tauri/resources/pocket-voices/marius.wav b/desktop/src-tauri/resources/pocket-voices/marius.wav new file mode 100644 index 0000000000..11ab460394 Binary files /dev/null and b/desktop/src-tauri/resources/pocket-voices/marius.wav differ diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index 94d162e620..cad26a5fca 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -48,15 +48,17 @@ pub struct AppState { pub channel_templates_store_lock: Mutex<()>, pub managed_agent_processes: Mutex>, pub huddle_state: Mutex, + pub tts_settings: Mutex, + pub tts_settings_load_error: Mutex>, + pub tts_settings_transition: tokio::sync::Mutex<()>, /// Tauri app handle — stored after setup so huddle commands can emit /// `huddle-state-changed` events without needing the handle threaded /// through every call site. - /// /// Set once during `setup()` in `lib.rs`; never cleared. pub app_handle: Mutex>, - /// Selected audio output device name. `None` = system default. - /// Used by `connect_audio_relay` and TTS pipeline when opening sinks. + /// Selected audio output device (`None` uses the system default) for relay and TTS. pub audio_output_device: Mutex>, + pub tts_playback_speed: crate::huddle::playback_speed::PlaybackSpeedControl, /// Port of the localhost media streaming proxy (set during setup). pub media_proxy_port: AtomicU16, /// Set when identity resolution detected a "keyring-locked" state: the @@ -213,8 +215,12 @@ pub fn build_app_state() -> AppState { managed_agent_processes: Mutex::new(HashMap::new()), session_config_cache: Mutex::new(HashMap::new()), huddle_state: Mutex::new(HuddleState::default()), + tts_settings: Mutex::new(Default::default()), + tts_settings_load_error: Mutex::new(None), + tts_settings_transition: tokio::sync::Mutex::new(()), app_handle: Mutex::new(None), audio_output_device: Mutex::new(None), + tts_playback_speed: crate::huddle::playback_speed::PlaybackSpeedControl::default(), media_proxy_port: AtomicU16::new(0), prevent_sleep: Arc::new(Mutex::new( crate::prevent_sleep::PreventSleepState::default(), diff --git a/desktop/src-tauri/src/huddle/agent_tts_routing_tests.rs b/desktop/src-tauri/src/huddle/agent_tts_routing_tests.rs new file mode 100644 index 0000000000..c6ce686853 --- /dev/null +++ b/desktop/src-tauri/src/huddle/agent_tts_routing_tests.rs @@ -0,0 +1,26 @@ +use super::{enqueue_agent_tts_text, normalize_agent_tts_text, MAX_TTS_TEXT_LEN}; + +#[tokio::test] +async fn assistant_plain_text_routes_unchanged_into_voice_pipeline_boundary() { + let (sender, receiver) = std::sync::mpsc::channel(); + let text = "A newly submitted assistant reply.".to_string(); + + enqueue_agent_tts_text(text.clone(), move |queued| { + sender.send(queued).map_err(|error| error.to_string()) + }) + .await + .expect("route assistant text"); + + assert_eq!(receiver.recv().expect("queued text"), text); +} + +#[test] +fn assistant_text_truncation_is_unicode_safe_before_voice_routing() { + let input = "🦀".repeat(MAX_TTS_TEXT_LEN + 1); + let output = normalize_agent_tts_text(input); + assert_eq!( + output.chars().count(), + MAX_TTS_TEXT_LEN + "... message truncated.".chars().count() + ); + assert!(output.ends_with("... message truncated.")); +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index a815bf2d06..bb1f69d5b3 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -28,6 +28,7 @@ pub mod audio_output; pub mod jitter; pub mod models; pub mod pipeline; +pub mod playback_speed; pub mod playout; pub mod pocket; pub mod preprocessing; @@ -37,6 +38,7 @@ pub mod state; pub mod stt; pub mod transcription; pub mod tts; +pub mod tts_settings; pub mod wire; // ── Shared utilities ────────────────────────────────────────────────────────── @@ -63,6 +65,7 @@ pub(super) fn drain_until_shutdown( pub use state::{HuddleJoinInfo, HuddlePhase, HuddleState, VoiceInputMode}; pub use transcription::{set_huddle_transcription_enabled, start_stt_pipeline}; +pub use tts_settings::set_tts_enabled; // ── Imports ─────────────────────────────────────────────────────────────────── @@ -817,53 +820,31 @@ pub fn get_model_status(_state: State<'_, AppState>) -> Result) -> Result<(), String> { - let old_pipeline = { - let mut hs = state.huddle()?; - hs.tts_enabled = enabled; - if !enabled { - hs.tts_pipeline.take() // Take out of lock. - } else { - None - } - }; - // Shut down outside the lock — thread join happens here. - if let Some(ref pipeline) = old_pipeline { - pipeline.shutdown(); - } - drop(old_pipeline); - - if enabled { - // Re-start TTS pipeline if models are available and huddle is active. - let phase = { - let hs = state.huddle()?; - hs.phase.clone() - }; - if matches!(phase, HuddlePhase::Connected | HuddlePhase::Active) { - if let Err(e) = maybe_start_tts_pipeline(&state).await { - eprintln!("buzz-desktop: TTS pipeline restart failed: {e}"); - } - } - } - - Ok(()) -} - /// Speak an agent message via TTS. /// /// Maximum text length accepted for TTS synthesis. /// ~2000 chars ≈ 1–2 minutes of speech. Longer messages are truncated. const MAX_TTS_TEXT_LEN: usize = 2000; +fn normalize_agent_tts_text(text: String) -> String { + if text.chars().count() > MAX_TTS_TEXT_LEN { + let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect(); + truncated.push_str("... message truncated."); + truncated + } else { + text + } +} + +async fn enqueue_agent_tts_text(text: String, enqueue: F) -> Result<(), String> +where + F: FnOnce(String) -> Result<(), String> + Send + 'static, +{ + tokio::task::spawn_blocking(move || enqueue(text)) + .await + .map_err(|error| format!("TTS enqueue task failed: {error}"))? +} + /// Called by the WebView when it receives an incoming agent kind:9 message. /// Lazily starts the TTS pipeline if models are ready but the pipeline hasn't /// been created yet (e.g. models finished downloading after huddle started). @@ -873,13 +854,7 @@ const MAX_TTS_TEXT_LEN: usize = 2000; pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Result<(), String> { // Truncate oversized messages — agents shouldn't monologue in a voice huddle. // Use char count (not byte length) to avoid panicking on multi-byte UTF-8. - let text = if text.chars().count() > MAX_TTS_TEXT_LEN { - let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect(); - truncated.push_str("... message truncated."); - truncated - } else { - text - }; + let text = normalize_agent_tts_text(text); let needs_pipeline = { let hs = state.huddle()?; @@ -895,15 +870,30 @@ pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Re } } - let hs = state.huddle()?; - if hs.tts_enabled { - if let Some(ref pipeline) = hs.tts_pipeline { - pipeline.speak(text)?; - } - } - Ok(()) + let sender = { + let hs = state.huddle()?; + hs.tts_enabled + .then(|| { + hs.tts_pipeline + .as_ref() + .map(|pipeline| pipeline.text_sender()) + }) + .flatten() + }; + let Some(sender) = sender else { + return Ok(()); + }; + enqueue_agent_tts_text(text, move |text| { + sender + .send(text) + .map_err(|error| format!("TTS queue closed while waiting to enqueue: {error}")) + }) + .await } +#[cfg(test)] +mod agent_tts_routing_tests; + /// Add an agent to the active huddle. /// /// Steps: diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 169ddf66c0..e9b4e02a2c 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -24,6 +24,9 @@ use std::sync::{Arc, Mutex, OnceLock}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +#[path = "models_voice_upgrade.rs"] +mod voice_upgrade; + // ── Integrity verification ──────────────────────────────────────────────────── // // All model artifacts are verified against pinned SHA-256 hashes before @@ -77,6 +80,7 @@ const TTS_FILE_HASHES: &[(&str, &str)] = &[ ("token_scores.json", "5be2f278caf9b9800741f0fd82bff677f4943ec764c356f907213434b622d958"), ("LICENSE", "fe7b4ce83b8381cc5b216bbb4af73c570688d1b819c73bbaed8ca401f4677cd6"), ("reference_sample.wav", "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f"), + ("marius.wav", "076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16"), ]; // ── Model versioning ────────────────────────────────────────────────────────── @@ -100,7 +104,7 @@ const STT_MODEL_VERSION: &str = "2"; /// reason explicit and skips the failing-then-re-fetching transient state. /// Bumped "2" → "3" for the int8 → fp32 model swap (see `POCKET_HF_BASE`): /// existing int8 installs must re-download the suffixless fp32 sessions. -const TTS_MODEL_VERSION: &str = "3"; +const TTS_MODEL_VERSION: &str = "4"; /// Filename for the version manifest written alongside model files. const MANIFEST_FILENAME: &str = ".buzz-model-manifest"; @@ -194,6 +198,14 @@ https://datashare.ed.ac.uk/handle/10283/3443 (CC-BY-4.0). Recording enhancement (denoise/dereverb) by ai-coustics: https://ai-coustics.com/ +Bundled reference voice (marius.wav): +\"Marius\", an audio-identical copy of Kyutai's +`voice-donations/Selfie.wav`, distributed under CC0 1.0 Universal: +https://huggingface.co/kyutai/tts-voices/blob/main/voice-donations/Selfie.wav +https://creativecommons.org/publicdomain/zero/1.0/ +The contributor submitted their own voice under Kyutai's Unmute Voice Donation +Project terms. Neither Kyutai nor the contributor endorses Buzz. + Buzz ships all ONNX/model artifacts and the reference voice WAV unmodified, renamed only by placement in the local model directory. @@ -212,6 +224,7 @@ const TTS_EXPECTED_FILES: &[&str] = &[ "token_scores.json", "LICENSE", "reference_sample.wav", + "marius.wav", TTS_LICENSE_FILE_NAME, ]; @@ -640,6 +653,9 @@ impl ModelManager { /// Start a background Pocket TTS download (~189 MB). No-op if already ready or downloading. pub fn start_tts_download(&self, http_client: reqwest::Client) { + if let Err(error) = voice_upgrade::install_marius_into_v3_model(&self.models_dir) { + eprintln!("buzz-desktop: could not upgrade existing Pocket voices in place: {error}"); + } let manager = self.clone(); self.tts.start_download( &self.models_dir, @@ -757,7 +773,7 @@ impl ModelManager { /// - five ONNX sessions (Pocket TTS + Mimi codec) /// - `vocab.json` / `token_scores.json` for sherpa-onnx text conditioning /// - upstream `LICENSE` plus Buzz's `MODEL_LICENSE.txt` attribution sidecar - /// - `reference_sample.wav` as the bundled default voice + /// - `reference_sample.wav` and embedded `marius.wav` reference voices /// /// Files are written to a temp directory first, then moved atomically. async fn download_tts_model(&self, http_client: reqwest::Client) -> Result<(), String> { @@ -845,6 +861,12 @@ impl ModelManager { tokio::fs::write(temp_dir.join(TTS_LICENSE_FILE_NAME), TTS_LICENSE_TEXT) .await .map_err(|e| format!("write TTS model license sidecar: {e}"))?; + tokio::fs::write( + temp_dir.join("marius.wav"), + voice_upgrade::POCKET_MARIUS_WAV, + ) + .await + .map_err(|e| format!("install bundled Marius voice: {e}"))?; self.tts.set_status(ModelStatus::Downloading { progress_percent: 90, diff --git a/desktop/src-tauri/src/huddle/models_voice_upgrade.rs b/desktop/src-tauri/src/huddle/models_voice_upgrade.rs new file mode 100644 index 0000000000..28c6d26da8 --- /dev/null +++ b/desktop/src-tauri/src/huddle/models_voice_upgrade.rs @@ -0,0 +1,73 @@ +use super::*; + +pub(super) const POCKET_MARIUS_WAV: &[u8] = + include_bytes!("../../resources/pocket-voices/marius.wav"); +const PRE_MARIUS_TTS_MODEL_VERSION: &str = "3"; + +/// Add Marius to an otherwise-ready v3 install without re-downloading models. +/// +/// The manifest is written last, so interruption leaves v3 intact and the next +/// launch retries. +pub(super) fn install_marius_into_v3_model(models_dir: &Path) -> Result<(), String> { + let model_dir = models_dir.join(TTS_MODEL_DIR_NAME); + let manifest_path = model_dir.join(MANIFEST_FILENAME); + let version = match std::fs::read_to_string(&manifest_path) { + Ok(version) => version, + Err(_) => return Ok(()), + }; + if version.trim() != PRE_MARIUS_TTS_MODEL_VERSION { + return Ok(()); + } + if !TTS_EXPECTED_FILES + .iter() + .filter(|filename| **filename != "marius.wav") + .all(|filename| model_dir.join(filename).is_file()) + { + return Ok(()); + } + + std::fs::write(model_dir.join("marius.wav"), POCKET_MARIUS_WAV) + .map_err(|error| format!("write Marius voice: {error}"))?; + std::fs::write(model_dir.join(TTS_LICENSE_FILE_NAME), TTS_LICENSE_TEXT) + .map_err(|error| format!("update Pocket voice notice: {error}"))?; + std::fs::write(manifest_path, TTS_MODEL_VERSION) + .map_err(|error| format!("update Pocket model manifest: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn v3_install_adds_marius_without_redownloading_models() { + let temp = tempfile::tempdir().expect("tempdir"); + let model_dir = temp.path().join(TTS_MODEL_DIR_NAME); + std::fs::create_dir_all(&model_dir).expect("create model dir"); + for file in TTS_EXPECTED_FILES + .iter() + .filter(|filename| **filename != "marius.wav") + { + std::fs::write(model_dir.join(file), b"existing").expect("write prior file"); + } + std::fs::write( + model_dir.join(MANIFEST_FILENAME), + PRE_MARIUS_TTS_MODEL_VERSION, + ) + .expect("write prior manifest"); + + install_marius_into_v3_model(temp.path()).expect("in-place upgrade"); + + assert_eq!( + std::fs::read(model_dir.join("marius.wav")).expect("Marius installed"), + POCKET_MARIUS_WAV + ); + assert_eq!( + std::fs::read_to_string(model_dir.join(MANIFEST_FILENAME)).expect("updated manifest"), + TTS_MODEL_VERSION + ); + assert!( + ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION) + .is_ready(temp.path()) + ); + } +} diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs index ceccedd8b6..68d873f4e1 100644 --- a/desktop/src-tauri/src/huddle/pipeline.rs +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -16,7 +16,7 @@ use crate::events; use super::models; use super::relay_api::{self, fetch_channel_members, parse_channel_uuid}; -use super::state::{HuddlePhase, VoiceInputMode}; +use super::state::{HuddlePhase, HuddleState, VoiceInputMode}; use super::stt; use super::tts; @@ -216,8 +216,23 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result Result Result { + let mut huddle = state.huddle()?; + huddle.tts_starting.store(false, Ordering::Release); + if !huddle.tts_enabled + || !matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active) + || huddle.tts_pipeline.is_some() { - let mut hs = state.huddle()?; - hs.tts_starting.store(false, Ordering::Release); - // Phase check: huddle may have been torn down during construction. - if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { - return Ok(false); - } - // Final check: another path may have created a pipeline while we were constructing. - if hs.tts_pipeline.is_some() { - return Ok(false); - } - hs.tts_pipeline = Some(pipeline); + return Ok(false); } - + let voice = state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}")) + .map(|settings| { + super::tts_settings::pocket_voice_name(&settings.voice_preferences).to_string() + })?; + publish(&voice, &mut huddle); Ok(true) } @@ -357,3 +390,98 @@ pub(crate) fn spawn_transcription_task( } }); } + +#[cfg(test)] +mod tts_start_race_tests { + use std::sync::{atomic::Ordering, Arc, Barrier, Mutex}; + + use crate::app_state::build_app_state; + + use super::{finalize_tts_pipeline_start, HuddlePhase}; + + #[test] + fn construction_reconciles_a_voice_selected_while_starting() { + let state = Arc::new(build_app_state()); + { + let mut huddle = state.huddle().expect("huddle state"); + huddle.phase = HuddlePhase::Active; + huddle.tts_enabled = true; + huddle.tts_starting.store(true, Ordering::Release); + } + + let constructed = Arc::new(Barrier::new(2)); + let publish = Arc::new(Barrier::new(2)); + let selected_voice = Arc::new(Mutex::new(None)); + let worker_state = Arc::clone(&state); + let worker_constructed = Arc::clone(&constructed); + let worker_publish = Arc::clone(&publish); + let worker_voice = Arc::clone(&selected_voice); + let worker = std::thread::spawn(move || { + worker_constructed.wait(); + worker_publish.wait(); + finalize_tts_pipeline_start(&worker_state, |voice, _| { + *worker_voice.lock().expect("selected voice") = Some(voice.to_string()); + }) + }); + + constructed.wait(); + assert!(state + .huddle() + .expect("huddle state") + .tts_starting + .load(Ordering::Acquire)); + state + .tts_settings + .lock() + .expect("text-to-speech settings") + .voice_preferences = vec!["pocket:marius".to_string()]; + publish.wait(); + + assert!(worker.join().expect("starter thread").expect("finalize")); + assert_eq!( + *selected_voice.lock().expect("selected voice"), + Some("marius".to_string()) + ); + } + + #[test] + fn construction_is_discarded_when_disabled_while_starting() { + let state = Arc::new(build_app_state()); + { + let mut huddle = state.huddle().expect("huddle state"); + huddle.phase = HuddlePhase::Active; + huddle.tts_enabled = true; + huddle.tts_starting.store(true, Ordering::Release); + } + + let constructed = Arc::new(Barrier::new(2)); + let publish = Arc::new(Barrier::new(2)); + let did_publish = Arc::new(Mutex::new(false)); + let worker_state = Arc::clone(&state); + let worker_constructed = Arc::clone(&constructed); + let worker_publish = Arc::clone(&publish); + let worker_did_publish = Arc::clone(&did_publish); + let worker = std::thread::spawn(move || { + worker_constructed.wait(); + worker_publish.wait(); + finalize_tts_pipeline_start(&worker_state, |_, _| { + *worker_did_publish.lock().expect("publish flag") = true; + }) + }); + + constructed.wait(); + { + let mut huddle = state.huddle().expect("huddle state"); + huddle.tts_enabled = false; + } + publish.wait(); + + assert!(!worker.join().expect("starter thread").expect("finalize")); + assert!(!*did_publish.lock().expect("publish flag")); + assert!(!state + .huddle() + .expect("huddle state") + .tts_starting + .load(Ordering::Acquire)); + } +} diff --git a/desktop/src-tauri/src/huddle/playback_speed.rs b/desktop/src-tauri/src/huddle/playback_speed.rs new file mode 100644 index 0000000000..ee1e845fca --- /dev/null +++ b/desktop/src-tauri/src/huddle/playback_speed.rs @@ -0,0 +1,331 @@ +//! Pitch-preserving playback speed for generated speech. +//! +//! This stage sits between Pocket synthesis and rodio playback. It is +//! deliberately independent of Pocket's model parameters, and it does not use +//! rodio's speed control because rodio changes pitch and speed together. + +use std::{ + io::Write, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, +}; + +use atomic_write_file::AtomicWriteFile; +use serde::{Deserialize, Serialize}; +use tauri::{AppHandle, Manager, State}; + +use crate::app_state::AppState; + +#[path = "playback_speed_dsp.rs"] +mod playback_speed_dsp; +pub(crate) use playback_speed_dsp::process_complete_chunk_preserving_lead_in; +use playback_speed_dsp::{validate_speed, DEFAULT_PLAYBACK_SPEED}; + +const SETTINGS_FILE: &str = "tts-playback-settings.json"; +#[cfg(test)] +use playback_speed_dsp::{process_complete_chunk, UNITY_EPSILON}; + +/// Lock-free shared control read by the TTS worker before each synthesis chunk. +#[derive(Clone, Debug)] +pub struct PlaybackSpeedControl { + speed: Arc, + transition: Arc>, +} + +impl Default for PlaybackSpeedControl { + fn default() -> Self { + Self { + speed: Arc::new(AtomicU32::new(DEFAULT_PLAYBACK_SPEED.to_bits())), + transition: Arc::new(tokio::sync::Mutex::new(())), + } + } +} + +impl PlaybackSpeedControl { + /// Return the current generated-speech playback speed. + pub fn get(&self) -> f32 { + f32::from_bits(self.speed.load(Ordering::Acquire)) + } + + /// Update the in-memory speed after validation. + pub fn set(&self, speed: f32) -> Result<(), String> { + validate_speed(speed)?; + self.speed.store(speed.to_bits(), Ordering::Release); + Ok(()) + } + + async fn persist_to_path(&self, path: &Path, speed: f32) -> Result<(), String> { + validate_speed(speed)?; + let _transition = self.transition.lock().await; + save_to_path(path, speed)?; + self.speed.store(speed.to_bits(), Ordering::Release); + Ok(()) + } +} + +#[derive(Debug, Deserialize, Serialize)] +struct PersistedPlaybackSettings { + speed: f32, +} + +/// Load the global playback speed during app setup. +pub fn load_playback_speed(app: &AppHandle, control: &PlaybackSpeedControl) -> Result<(), String> { + let path = settings_path(app)?; + let speed = load_from_path(&path)?; + control.set(speed) +} + +/// Return the globally configured generated-speech playback speed. +#[tauri::command] +pub fn get_tts_playback_speed(state: State<'_, AppState>) -> f32 { + state.tts_playback_speed.get() +} + +/// Persist and apply the global generated-speech playback speed. +#[tauri::command] +pub async fn set_tts_playback_speed( + speed: f32, + app: AppHandle, + state: State<'_, AppState>, +) -> Result<(), String> { + state + .tts_playback_speed + .persist_to_path(&settings_path(&app)?, speed) + .await +} + +fn settings_path(app: &AppHandle) -> Result { + app.path() + .app_data_dir() + .map(|directory| directory.join(SETTINGS_FILE)) + .map_err(|error| format!("resolve TTS playback settings directory: {error}")) +} + +fn load_from_path(path: &Path) -> Result { + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(DEFAULT_PLAYBACK_SPEED); + } + Err(error) => return Err(format!("read {}: {error}", path.display())), + }; + let settings: PersistedPlaybackSettings = serde_json::from_slice(&bytes) + .map_err(|error| format!("parse {}: {error}", path.display()))?; + validate_speed(settings.speed)?; + Ok(settings.speed) +} + +fn save_to_path(path: &Path, speed: f32) -> Result<(), String> { + validate_speed(speed)?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| format!("create {}: {error}", parent.display()))?; + } + let payload = serde_json::to_vec_pretty(&PersistedPlaybackSettings { speed }) + .map_err(|error| format!("serialize TTS playback settings: {error}"))?; + let resolved = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let mut file = AtomicWriteFile::open(&resolved) + .map_err(|error| format!("open {} for atomic write: {error}", resolved.display()))?; + file.write_all(&payload) + .map_err(|error| format!("write {}: {error}", resolved.display()))?; + file.commit() + .map_err(|error| format!("commit {}: {error}", resolved.display())) +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProcessorKind { + Bypass, + Signalsmith, +} + +#[cfg(test)] +fn processor_kind(speed: f32) -> ProcessorKind { + if (speed - DEFAULT_PLAYBACK_SPEED).abs() <= UNITY_EPSILON { + ProcessorKind::Bypass + } else { + ProcessorKind::Signalsmith + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_RATE: u32 = 24_000; + + #[test] + fn selects_bypass_only_at_unity() { + assert_eq!(processor_kind(1.0), ProcessorKind::Bypass); + assert_eq!(processor_kind(1.25), ProcessorKind::Signalsmith); + } + + #[test] + fn complete_processing_preserves_length_pitch_and_order() { + let input: Vec = (0..24_000) + .map(|sample| { + let frequency = if sample < 8_000 { + 220.0 + } else if sample < 16_000 { + 440.0 + } else { + 660.0 + }; + (2.0 * std::f32::consts::PI * frequency * sample as f32 / SAMPLE_RATE as f32).sin() + }) + .collect(); + let output = process_complete_chunk(&input, 1.25, SAMPLE_RATE).expect("complete output"); + + assert_eq!(output.len(), 19_200); + let first_frequency = zero_crossing_frequency(&output[1_500..5_000], SAMPLE_RATE); + let second_frequency = zero_crossing_frequency(&output[7_500..11_000], SAMPLE_RATE); + let third_frequency = zero_crossing_frequency(&output[14_000..18_000], SAMPLE_RATE); + assert!( + (first_frequency - 220.0).abs() < 8.0, + "first segment measured {first_frequency} Hz" + ); + assert!( + (second_frequency - 440.0).abs() <= 10.0, + "second segment measured {second_frequency} Hz" + ); + assert!( + (third_frequency - 660.0).abs() <= 12.0, + "third segment measured {third_frequency} Hz" + ); + } + + #[test] + fn processor_preserves_sine_pitch() { + let frequency = 220.0_f32; + let input: Vec = (0..SAMPLE_RATE * 2) + .map(|sample| { + (2.0 * std::f32::consts::PI * frequency * sample as f32 / SAMPLE_RATE as f32).sin() + }) + .collect(); + let output = process_complete_chunk(&input, 1.5, SAMPLE_RATE).expect("complete output"); + assert!( + root_mean_square(&output[..480]) > 0.2, + "full reset pre-roll was not removed" + ); + assert!( + root_mean_square(&output[output.len() - 480..]) > 0.2, + "speech tail was truncated" + ); + let measured = zero_crossing_frequency(&output[2_000..], SAMPLE_RATE); + assert!( + (measured - frequency).abs() < 3.0, + "expected {frequency} Hz, measured {measured} Hz" + ); + } + + #[test] + fn complete_processing_preserves_onset_timing() { + let input: Vec = (0..SAMPLE_RATE) + .map(|sample| { + if (4_800..12_000).contains(&sample) || sample >= 16_800 { + (2.0 * std::f32::consts::PI * 220.0 * sample as f32 / SAMPLE_RATE as f32).sin() + } else { + 0.0 + } + }) + .collect(); + let output = process_complete_chunk(&input, 1.25, SAMPLE_RATE).expect("complete output"); + let active_windows = active_window_starts(&output, 240, 0.15); + + let first_onset = *active_windows.first().expect("first tone onset"); + let second_onset = active_windows + .windows(2) + .find_map(|pair| (pair[1] > pair[0] + 480).then_some(pair[1])) + .expect("second tone onset"); + assert!( + (3_200..=4_400).contains(&first_onset), + "first onset shifted to sample {first_onset}" + ); + assert!( + (12_800..=14_000).contains(&second_onset), + "second onset shifted to sample {second_onset}" + ); + } + + #[test] + fn non_unity_latency_stays_below_75_ms() { + let mut stretch = ssstretch::Stretch::new(); + stretch.preset_default(1, SAMPLE_RATE as f32); + let latency_ms = stretch.output_latency().max(0) as f64 * 1_000.0 / SAMPLE_RATE as f64; + assert!(latency_ms <= 75.0, "algorithmic latency was {latency_ms}ms"); + } + + #[test] + fn persisted_speed_round_trips_and_rejects_invalid_values() { + let directory = tempfile::tempdir().expect("tempdir"); + let path = directory.path().join(SETTINGS_FILE); + save_to_path(&path, 1.25).expect("save"); + assert_eq!(load_from_path(&path).expect("load"), 1.25); + + std::fs::write(&path, br#"{"speed":2.0}"#).expect("invalid fixture"); + assert!(load_from_path(&path).is_err()); + } + + #[tokio::test] + async fn serialized_persistence_finishes_with_the_latest_speed() { + let directory = tempfile::tempdir().expect("tempdir"); + let path = directory.path().join(SETTINGS_FILE); + let control = PlaybackSpeedControl::default(); + let transition = control.transition.lock().await; + + let (first_enqueued_tx, first_enqueued_rx) = tokio::sync::oneshot::channel(); + let first_control = control.clone(); + let first_path = path.clone(); + let first = tokio::spawn(async move { + first_enqueued_tx.send(()).expect("signal first waiter"); + first_control.persist_to_path(&first_path, 1.25).await + }); + first_enqueued_rx.await.expect("first waiter started"); + tokio::task::yield_now().await; + + let (second_enqueued_tx, second_enqueued_rx) = tokio::sync::oneshot::channel(); + let second_control = control.clone(); + let second_path = path.clone(); + let second = tokio::spawn(async move { + second_enqueued_tx.send(()).expect("signal second waiter"); + second_control.persist_to_path(&second_path, 1.5).await + }); + second_enqueued_rx.await.expect("second waiter started"); + tokio::task::yield_now().await; + + drop(transition); + first.await.expect("join first save").expect("first save"); + second + .await + .expect("join second save") + .expect("second save"); + + assert_eq!(load_from_path(&path).expect("load final speed"), 1.5); + assert_eq!(control.get(), 1.5); + } + + fn zero_crossing_frequency(samples: &[f32], sample_rate: u32) -> f32 { + let crossings = samples + .windows(2) + .filter(|pair| pair[0] <= 0.0 && pair[1] > 0.0) + .count(); + crossings as f32 * sample_rate as f32 / samples.len() as f32 + } + + fn root_mean_square(samples: &[f32]) -> f32 { + (samples.iter().map(|sample| sample * sample).sum::() / samples.len() as f32).sqrt() + } + + fn active_window_starts(samples: &[f32], window: usize, threshold: f32) -> Vec { + samples + .chunks_exact(window) + .enumerate() + .filter_map(|(index, chunk)| { + (root_mean_square(chunk) > threshold).then_some(index * window) + }) + .collect() + } +} diff --git a/desktop/src-tauri/src/huddle/playback_speed_dsp.rs b/desktop/src-tauri/src/huddle/playback_speed_dsp.rs new file mode 100644 index 0000000000..232a2963a3 --- /dev/null +++ b/desktop/src-tauri/src/huddle/playback_speed_dsp.rs @@ -0,0 +1,116 @@ +//! Pure pitch-preserving playback-speed processing. +//! +//! This module intentionally has no application, persistence, or filesystem +//! dependencies so production playback and diagnostic examples can compile +//! the exact same DSP boundary. + +/// Slowest supported generated-speech playback speed. +pub const MIN_PLAYBACK_SPEED: f32 = 0.75; +/// Fastest supported generated-speech playback speed. +pub const MAX_PLAYBACK_SPEED: f32 = 1.5; +/// Default generated-speech playback speed. +pub const DEFAULT_PLAYBACK_SPEED: f32 = 1.0; + +pub(super) const UNITY_EPSILON: f32 = 0.000_1; + +/// Pitch-preserve one already-buffered Pocket chunk. +/// +/// The returned buffer has exactly `input.len() / speed` samples. Signalsmith +/// starts a reset processor `input_latency` samples before the supplied audio +/// and emits another `output_latency` samples of pre-roll. Both components are +/// removed after draining, so the returned chunk retains its beginning and +/// tail without buffering any later Pocket chunk. +pub fn process_complete_chunk( + input: &[f32], + speed: f32, + sample_rate: u32, +) -> Result, String> { + validate_speed(speed)?; + if input.is_empty() || (speed - DEFAULT_PLAYBACK_SPEED).abs() <= UNITY_EPSILON { + return Ok(input.to_vec()); + } + + let expected = (input.len() as f64 / speed as f64).round() as usize; + let mut stretch = ssstretch::Stretch::new(); + stretch.preset_default(1, sample_rate as f32); + let input_latency = stretch.input_latency().max(0) as usize; + let output_latency = stretch.output_latency().max(0) as usize; + let reset_pre_roll = (input_latency as f64 / speed as f64).ceil() as usize; + + let inputs = [input.to_vec()]; + let mut output = [Vec::with_capacity(expected)]; + stretch.process_vec( + &inputs, + i32_len(input.len())?, + &mut output, + i32_len(expected)?, + ); + + let latency_input = [vec![0.0; input_latency]]; + let mut latency_output = [Vec::with_capacity(reset_pre_roll)]; + stretch.process_vec( + &latency_input, + i32_len(input_latency)?, + &mut latency_output, + i32_len(reset_pre_roll)?, + ); + output[0].extend_from_slice(&latency_output[0]); + + let mut flushed = [Vec::with_capacity(output_latency)]; + stretch.flush_vec(&mut flushed, i32_len(output_latency)?); + output[0].extend_from_slice(&flushed[0]); + + let start = reset_pre_roll.saturating_add(output_latency); + let end = start.saturating_add(expected); + if output[0].len() < end { + return Err(format!( + "time stretcher produced {} samples, need {end}", + output[0].len() + )); + } + Ok(output[0][start..end].to_vec()) +} + +/// Preserve a fixed device lead-in while pitch-preserving the remaining audio. +pub(crate) fn process_complete_chunk_preserving_lead_in( + input: &[f32], + fixed_lead_in_samples: usize, + speed: f32, + sample_rate: u32, +) -> Result, String> { + if fixed_lead_in_samples > input.len() { + return Err(format!( + "fixed lead-in of {fixed_lead_in_samples} samples exceeds input length {}", + input.len() + )); + } + + let processed = process_complete_chunk(&input[fixed_lead_in_samples..], speed, sample_rate)?; + let mut output = Vec::with_capacity(fixed_lead_in_samples + processed.len()); + output.extend_from_slice(&input[..fixed_lead_in_samples]); + output.extend_from_slice(&processed); + Ok(output) +} + +/// Return Signalsmith's compensated output lookahead for descriptive reporting. +#[allow(dead_code)] +pub(crate) fn compensated_output_latency_samples(sample_rate: u32) -> usize { + let mut stretch = ssstretch::Stretch::new(); + stretch.preset_default(1, sample_rate as f32); + stretch.output_latency().max(0) as usize +} + +/// Validate a generated-speech playback speed. +pub fn validate_speed(speed: f32) -> Result<(), String> { + if speed.is_finite() && (MIN_PLAYBACK_SPEED..=MAX_PLAYBACK_SPEED).contains(&speed) { + Ok(()) + } else { + Err(format!( + "Speech playback speed must be between {MIN_PLAYBACK_SPEED} and {MAX_PLAYBACK_SPEED}" + )) + } +} + +fn i32_len(length: usize) -> Result { + i32::try_from(length).map_err(|_| "audio chunk is too large to process".to_string()) +} diff --git a/desktop/src-tauri/src/huddle/pocket.rs b/desktop/src-tauri/src/huddle/pocket.rs index ee1faf928a..8e19781eeb 100644 --- a/desktop/src-tauri/src/huddle/pocket.rs +++ b/desktop/src-tauri/src/huddle/pocket.rs @@ -1,654 +1,3 @@ -//! Pocket TTS engine wrapper around sherpa-onnx's `OfflineTts`. -//! -//! Pocket TTS is a small (~473 MB fp32 ONNX) zero-shot voice-cloning TTS -//! model from Kyutai. It runs quickly on CPU via sherpa-onnx, replacing the -//! previous Kokoro-82M engine that also required an espeak-free but -//! lexicon-heavy G2P pipeline (Misaki + CMUdict). -//! -//! Full-precision fp32 sessions, not the ~189 MB int8 quantization we -//! originally shipped: a direct same-runtime A/B (k2-fsa/sherpa-onnx#3172) -//! found the int8 ONNX export audibly degraded output quality, and fp32 -//! "significantly improved quality even at 1 step". -//! -//! ## Attribution -//! -//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. -//! arXiv:2509.06926. Original repository: . -//! Licensed CC-BY-4.0. -//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. -//! - **ONNX export**: KevinAHM — -//! . CC-BY-4.0. -//! - **sherpa-onnx repackage**: csukuangfj / k2-fsa — -//! . -//! Repackages KevinAHM's export with the file layout sherpa-onnx's -//! `OfflineTtsPocketModelConfig` expects. CC-BY-4.0. -//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary -//! (f, conversation)" preset from the Kyutai TTS demo -//! (), which maps to `vctk/p333_023_enhanced.wav` -//! in . CC-BY-4.0, base recording -//! from the VCTK corpus, enhanced by ai-coustics. -//! -//! Buzz ships these files unmodified; see the on-disk `MODEL_LICENSE.txt` -//! sidecar written by `huddle::models` during install for the canonical -//! CC-BY-4.0 §3(a)(1) attribution block. -//! -//! ## Engine-module contract (see `huddle::tts`) -//! -//! `pocket.rs` exposes a fixed surface used by `tts.rs`. Mirroring this -//! contract is what lets the TTS pipeline stay engine-agnostic: -//! -//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. -//! - `DEFAULT_VOICE: &str` — default voice name (without extension). -//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. -//! - `load_text_to_speech(model_dir)` → `Result` -//! - `load_voice_style(path)` → `Result` -//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` -//! → `Result, String>` -//! -//! `lang` and `steps` are accepted for API compatibility with the previous -//! Kokoro engine but are unused — Pocket TTS does its own language ID from -//! the input text and is not a diffusion model (consistency LM, one step). -//! There is no speed knob: sherpa-onnx's `GenerationConfig.speed` is only -//! read by some model families (vits), never by the Pocket impl -//! (`offline-tts-pocket-impl.h` — zero references), and upstream pocket-tts -//! has no speed parameter either. +//! Compatibility re-export for the shared Pocket TTS engine. -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; - -// ── Engine-module contract: public consts ───────────────────────────────────── - -/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, -/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. -pub const SAMPLE_RATE: u32 = 24_000; - -/// Name (without extension) of the bundled reference voice. The model directory -/// is expected to contain `.` after install. -pub const DEFAULT_VOICE: &str = "reference_sample"; - -/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the -/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, -/// not a precomputed embedding, so the extension change is honest. -pub const VOICE_FILE_EXT: &str = "wav"; - -// ── Tuning ──────────────────────────────────────────────────────────────────── - -/// Single-threaded ONNX execution for predictable CPU contention with the STT -/// pipeline. Matches `STT_NUM_THREADS` in `stt.rs`; raise only if a benchmark -/// argues for it. -const TTS_NUM_THREADS: i32 = 1; - -/// LRU cache size for cloned voice embeddings inside the sherpa-onnx engine. -/// We bind to one voice per pipeline today, but the upstream example uses 16 -/// and the cost is negligible — keep room for future multi-voice support. -const VOICE_EMBEDDING_CACHE_CAPACITY: i32 = 16; - -/// Pocket TTS is a consistency-based LM. Generation quality saturates at one -/// denoising step — the upstream `GenerationConfig` default of 5 multiplies -/// synthesis time by ~5× with no audible benefit on this model. -const SYNTH_NUM_STEPS: i32 = 1; - -/// Leave the generated audio's silences untouched (1.0 is the identity). -/// -/// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding -/// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and -/// multiplies its length by this factor. The previous value of 0.0 — set -/// under the mistaken belief it disabled lead-in/lead-out padding — deleted -/// every natural pause inside an utterance: clause breaks, breaths, the gap -/// after a comma. Words slammed together and endings cut abruptly. The -/// reference Pocket TTS pipeline does not post-process silence at all; -/// 1.0 restores parity. -const SYNTH_SILENCE_SCALE: f32 = 1.0; - -/// sherpa-onnx upstream default for `max_frames` (LM steps), in -/// `offline-tts-pocket-impl.h:Generate`. 500 steps ≈ 40 s of audio at the -/// Mimi 12.5 Hz frame rate. Referenced only by the regression test below; -/// production code path never raises (or even reads) this value — we just -/// leave sherpa-onnx's own default in place by not setting the override. -#[cfg(test)] -const SHERPA_ONNX_MAX_FRAMES_DEFAULT: i32 = 500; - -/// Tight `max_frames` we ask for on short, padded prompts to bound the -/// original "monster breathing" runaway. 100 LM steps ≈ 8 s of audio — -/// roomy for any one-to-four-word utterance the user is likely to elicit -/// while still well short of the 40 s upstream default. Chosen with slack so -/// we never *truncate* a legitimate short reply. -const SHORT_PROMPT_MAX_FRAMES: i32 = 100; - -/// Word-count threshold (inclusive) below which we pad the prompt with -/// leading spaces and cap `max_frames` tighter than the upstream default. -/// Matches upstream `pocket_tts.models.tts_model.prepare_text_prompt`. Above -/// this threshold we leave sherpa-onnx's own defaults in place — overriding -/// them caused the "first 'yep' is just static" regression seen on -/// 2026-05-18, where dropping `frames_after_eos` below the upstream default -/// of 3 clipped the leading audio of multi-clause sentences. -const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; - -/// Number of leading spaces prepended to short prompts. The upstream Python -/// uses exactly 8 — keep parity rather than tuning blindly. -/// -/// This is upstream's *only* mitigation for the FlowLM cold-start smear on -/// short utterances (kyutai-labs/pocket-tts #91, #70): the autoregressive -/// generation has a 2–3 step "settle" period where the first phoneme can be -/// smeared. A previous revision added a sacrificial `". . "` prefix plus an -/// amplitude-threshold trim to strip the rendered prefix from the output — -/// but the trim's absolute threshold (0.02 against raw peaks of ~0.076) sat -/// in soft-onset territory and could eat real word starts, and its tuning -/// was calibrated against `silence_scale = 0.0` audio. Deleted in favour of -/// upstream parity: accept the occasional smeared first syllable rather -/// than risk trimming real speech. -const SHORT_PROMPT_PAD_SPACES: usize = 8; - -/// sherpa-onnx's documented `frames_after_eos` default. We deliberately do -/// *not* override this knob — the previous attempt to bump it for short -/// inputs and lower it for long inputs lowered it below the upstream default -/// of 3, which clipped the leading audio of multi-clause sentences (the -/// "first 'yep' is static" regression). The constant exists only for the -/// regression test below. Source: `offline-tts-pocket-impl.h:Generate`. -#[cfg(test)] -const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; - -// ── ONNX file names (five Pocket TTS sessions plus two JSON tables) ─────────── - -const FILE_LM_MAIN: &str = "lm_main.onnx"; -const FILE_LM_FLOW: &str = "lm_flow.onnx"; -const FILE_ENCODER: &str = "encoder.onnx"; -const FILE_DECODER: &str = "decoder.onnx"; -const FILE_TEXT_COND: &str = "text_conditioner.onnx"; -const FILE_VOCAB: &str = "vocab.json"; -const FILE_TOKEN_SCORES: &str = "token_scores.json"; - -// ── Voice style ─────────────────────────────────────────────────────────────── - -/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. -/// -/// Pocket TTS takes a reference waveform per generation call (not a -/// precomputed style embedding), so we keep the samples in memory and clone -/// the small `Vec` into each `GenerationConfig` rather than re-reading the -/// WAV from disk on every sentence. -#[derive(Debug, Clone)] -pub struct VoiceStyle { - samples: Vec, - sample_rate: i32, -} - -/// Load a reference voice WAV from disk. -/// -/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS -/// resamples internally using `reference_sample_rate`. The bundled -/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. -pub fn load_voice_style(path: &Path) -> Result { - let path_str = path - .to_str() - .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; - let wave = Wave::read(path_str) - .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; - let samples = wave.samples().to_vec(); - if samples.is_empty() { - return Err(format!("voice WAV is empty: {}", path.display())); - } - Ok(VoiceStyle { - samples, - sample_rate: wave.sample_rate(), - }) -} - -// ── Engine ──────────────────────────────────────────────────────────────────── - -/// Pocket TTS engine handle. Cheap to construct (one `OfflineTts::create` -/// call). Owned by the TTS worker thread for the lifetime of a huddle session. -/// -/// `OfflineTts` does not implement `Debug`, so we don't derive it here — the -/// pipeline only needs to move the engine into the worker thread and call -/// `synth_chunk` on it, never to print it. -pub struct PocketTts { - inner: OfflineTts, -} - -/// Build the Pocket TTS engine from the model directory installed by -/// `huddle::models`. Returns `Err` if any expected ONNX or JSON file is -/// missing — readiness is normally enforced by `is_tts_ready` upstream, but -/// the check is repeated here so a manually-modified model dir produces a -/// clear error string instead of an opaque sherpa-onnx `None`. -pub fn load_text_to_speech(model_dir: &str) -> Result { - let dir = PathBuf::from(model_dir); - for name in [ - FILE_LM_MAIN, - FILE_LM_FLOW, - FILE_ENCODER, - FILE_DECODER, - FILE_TEXT_COND, - FILE_VOCAB, - FILE_TOKEN_SCORES, - ] { - let p = dir.join(name); - if !p.is_file() { - return Err(format!("missing Pocket TTS file: {}", p.display())); - } - } - - let to_str = |name: &str| -> String { dir.join(name).to_string_lossy().into_owned() }; - - // Build the config by mutating defaults — mirrors `stt.rs` and stays - // resilient if sherpa-onnx adds unrelated model-family fields. - let mut cfg = OfflineTtsConfig::default(); - cfg.model.pocket.lm_main = Some(to_str(FILE_LM_MAIN)); - cfg.model.pocket.lm_flow = Some(to_str(FILE_LM_FLOW)); - cfg.model.pocket.encoder = Some(to_str(FILE_ENCODER)); - cfg.model.pocket.decoder = Some(to_str(FILE_DECODER)); - cfg.model.pocket.text_conditioner = Some(to_str(FILE_TEXT_COND)); - cfg.model.pocket.vocab_json = Some(to_str(FILE_VOCAB)); - cfg.model.pocket.token_scores_json = Some(to_str(FILE_TOKEN_SCORES)); - cfg.model.pocket.voice_embedding_cache_capacity = VOICE_EMBEDDING_CACHE_CAPACITY; - cfg.model.num_threads = TTS_NUM_THREADS; - // Explicit — defaults are not part of the API contract, and noisy debug - // logging in release builds would be expensive on every synthesized chunk. - cfg.model.debug = false; - - let inner = OfflineTts::create(&cfg) - .ok_or_else(|| "OfflineTts::create returned None for Pocket TTS".to_string())?; - Ok(PocketTts { inner }) -} - -// ── Prompt preparation ──────────────────────────────────────────────────────── - -/// Result of [`prepare_pocket_prompt`]: a synthesizer-ready prompt plus the -/// per-call generation overrides derived from the original text. -/// -/// `None` for either override means "leave sherpa-onnx's documented default -/// in place". The pipeline only sets `max_frames` (and only for short -/// padded inputs) so it can bound the original "monster breathing" runaway -/// without disturbing the rest of the LM sampling envelope. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct PreparedPrompt { - /// Text to hand to `OfflineTts::generate_with_config`. Capitalized, - /// punctuation-terminated, and (for short inputs) left-padded with - /// spaces — upstream's mitigation for the FlowLM cold-start smear. - pub text: String, - /// Value to pass via `GenerationConfig.extra["max_frames"]`, or `None` to - /// keep the upstream default of 500 LM steps. We only override on short - /// padded prompts where we have a tight expectation on output length. - pub max_frames: Option, -} - -/// Mirror of the *text-preparation* half of upstream -/// `pocket_tts.models.tts_model.prepare_text_prompt`. Sherpa-onnx's C++ -/// Pocket TTS impl does not run these preparation steps, so short / -/// unpunctuated / lowercase inputs can trigger up to 40 s of runaway -/// generation when the EOS logit never crosses its threshold. We replicate -/// the upstream Python recipe here: -/// -/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but -/// cheap to re-check after sentence splitting). -/// 2. Capitalize the first letter. -/// 3. Append `.` if the text doesn't end in punctuation. -/// 4. If fewer than five words, prepend `SHORT_PROMPT_PAD_SPACES` spaces -/// (upstream's cold-start mitigation — see the constant's docstring) and -/// return a tight [`SHORT_PROMPT_MAX_FRAMES`] cap so the LM can't run -/// away if EOS still doesn't fire. -/// -/// We do **not** override `frames_after_eos` — sherpa-onnx's default of 3 -/// is what we want. An earlier version set it to 1 on long inputs, which -/// clipped the leading audio of multi-clause sentences ("first 'yep' is -/// just static" regression). Tests `prepare_prompt_never_lowers_frames_…` -/// lock this in. -/// -/// Returns `None` only if the input is empty after trimming — caller should -/// skip synthesis in that case. -pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { - let trimmed = input.trim(); - if trimmed.is_empty() { - return None; - } - - // Collapse stray double-spaces / embedded newlines that may slip past - // `preprocess_for_tts` when sentences are spliced back together. - let mut cleaned = String::with_capacity(trimmed.len()); - let mut last_was_space = false; - for ch in trimmed.chars() { - let is_ws = ch.is_whitespace(); - if is_ws { - if !last_was_space { - cleaned.push(' '); - } - last_was_space = true; - } else { - cleaned.push(ch); - last_was_space = false; - } - } - - // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). - let first = cleaned.chars().next().expect("cleaned non-empty above"); - if first.is_lowercase() { - let upper: String = first.to_uppercase().collect(); - let mut iter = cleaned.chars(); - iter.next(); - cleaned = upper + iter.as_str(); - } - - // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. - // The upstream Python only checks `isalnum` → period, but for our agent - // text we already may end in `!` `?` `.` etc. — treat any of those as OK. - let last = cleaned - .chars() - .next_back() - .expect("cleaned non-empty above"); - if !matches!(last, '.' | '!' | '?' | ';' | ':' | ',') { - cleaned.push('.'); - } - - // Word count of the *cleaned but not padded* text — padding is whitespace - // only and would just lie to the threshold check below. - let word_count = cleaned.split_whitespace().count(); - - let (final_text, max_frames) = if word_count <= SHORT_PROMPT_WORD_THRESHOLD { - let mut padded = String::with_capacity(cleaned.len() + SHORT_PROMPT_PAD_SPACES); - for _ in 0..SHORT_PROMPT_PAD_SPACES { - padded.push(' '); - } - padded.push_str(&cleaned); - (padded, Some(SHORT_PROMPT_MAX_FRAMES)) - } else { - // For everything ≥5 words, fall back to upstream defaults. Overriding - // these is what caused the "first 'yep' is static" regression — the - // upstream LM has been tuned for `frames_after_eos = 3` and - // `max_frames = 500`, and there's no clear win in second-guessing. - (cleaned, None) - }; - - Some(PreparedPrompt { - text: final_text, - max_frames, - }) -} - -/// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. -/// -/// Centralised so the regression test below can assert that we **never** -/// emit a `frames_after_eos` override — the previous attempt to override -/// that knob (setting it to 1 for ≥5-word inputs) clipped the leading -/// audio of multi-clause sentences (the "first 'yep' is static" bug on -/// 2026-05-18). The upstream sherpa-onnx default of 3 is what we want, and -/// the right way to keep it is to not set it at all. -fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { - prepared.max_frames.map(|mf| { - let mut h: HashMap = HashMap::with_capacity(1); - h.insert("max_frames".to_string(), serde_json::Value::from(mf)); - h - }) -} - -impl PocketTts { - /// Synthesise `text` with the given reference voice. - /// - /// `_lang` and `_steps` are accepted for API compatibility with the - /// previous Kokoro engine. Pocket TTS infers language from the input text - /// directly and is a one-step consistency model. Returns an empty buffer - /// for whitespace-only input. - pub fn synth_chunk( - &self, - text: &str, - _lang: &str, - style: &VoiceStyle, - _steps: usize, - ) -> Result, String> { - // Mirror upstream pocket-tts prompt prep — without this short or - // unpunctuated inputs can cause the LM's EOS logit to never trip, - // producing up to 40 s of "monster breathing" garbage on the first - // utterance. See `prepare_pocket_prompt` for the full recipe. - let prepared = match prepare_pocket_prompt(text) { - Some(p) => p, - None => return Ok(Vec::new()), - }; - - // Per-call generation hints sherpa-onnx forwards to - // `offline-tts-pocket-impl.h`. We only override `max_frames`, and - // only for short padded prompts where we have a tight expectation - // on output length — that bounds the original runaway without - // disturbing the rest of the LM sampling envelope. See - // `prepare_pocket_prompt` docs for the regression history. - let extra = build_generation_extra(&prepared); - - let cfg = GenerationConfig { - num_steps: SYNTH_NUM_STEPS, - silence_scale: SYNTH_SILENCE_SCALE, - reference_audio: Some(style.samples.clone()), - reference_sample_rate: style.sample_rate, - extra, - // `speed` stays at its default: the Pocket impl never reads it - // (see the engine-contract note in the module docs). - ..Default::default() - }; - - // No progress callback — synthesis is fast enough that returning the - // whole buffer at once keeps the lookahead pipelining in `tts.rs` - // simple. `None:: bool>` pins the callback type for the - // `generate_with_config` generic parameter. - let audio = self - .inner - .generate_with_config(&prepared.text, &cfg, None:: bool>) - .ok_or_else(|| { - format!( - "Pocket TTS synthesis failed for text ({} chars)", - prepared.text.len() - ) - })?; - - let sample_rate = audio.sample_rate(); - if sample_rate != SAMPLE_RATE as i32 { - eprintln!( - "buzz-desktop: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ - (expected {SAMPLE_RATE}Hz); playback speed may be wrong" - ); - } - - Ok(audio.samples().to_vec()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // ── prepare_pocket_prompt ──────────────────────────────────────────────── - - #[test] - fn prepare_prompt_returns_none_for_empty_input() { - assert!(prepare_pocket_prompt("").is_none()); - assert!(prepare_pocket_prompt(" ").is_none()); - assert!(prepare_pocket_prompt("\n\t ").is_none()); - } - - /// Helper: the exact leading sequence prepended to every short prompt — - /// 8 spaces of padding (upstream's cold-start mitigation). - /// Centralising this keeps the assertions readable. - fn short_prefix() -> String { - " ".repeat(SHORT_PROMPT_PAD_SPACES) - } - - #[test] - fn prepare_prompt_pads_and_capitalizes_one_word() { - // The "yep" case Tyler hit in production — bare lowercase one-word - // utterance with no punctuation. Must be padded with the short-prompt - // space pad, capitalized, terminated, with a tight `max_frames` cap - // to bound runaway gen. - let out = prepare_pocket_prompt("yep").expect("non-empty"); - assert_eq!(out.text, format!("{}Yep.", short_prefix())); - assert_eq!(out.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - const { - assert!( - SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT, - "short cap must be tighter than the upstream default" - ); - } - } - - #[test] - fn prepare_prompt_preserves_existing_punctuation() { - let out = prepare_pocket_prompt("yes!").expect("non-empty"); - assert_eq!(out.text, format!("{}Yes!", short_prefix())); // exclamation kept - let out = prepare_pocket_prompt("really?").expect("non-empty"); - assert_eq!(out.text, format!("{}Really?", short_prefix())); - } - - #[test] - fn prepare_prompt_threshold_is_inclusive_at_four_words() { - // 4 words = short (padded + tight max_frames); 5 words = long - // (no padding, no overrides — upstream defaults stand). - let four = prepare_pocket_prompt("one two three four").expect("non-empty"); - assert_eq!( - four.text, - format!("{}One two three four.", short_prefix()), - "four-word input should get exactly the space pad" - ); - assert_eq!(four.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - - let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); - assert!( - !five.text.starts_with(' '), - "five-word input should NOT be padded" - ); - assert_eq!( - five.max_frames, None, - "long inputs must leave sherpa-onnx's max_frames default in place" - ); - } - - #[test] - fn prepare_prompt_does_not_pad_long_text() { - let long = "This is a longer sentence that the model should handle just fine."; - let out = prepare_pocket_prompt(long).expect("non-empty"); - assert!(!out.text.starts_with(' ')); - assert_eq!(out.max_frames, None); - assert!(out.text.ends_with('.')); - } - - #[test] - fn prepare_prompt_collapses_whitespace() { - let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); - // 3 words → short → padded. Interior whitespace collapsed. - assert_eq!(out.text, format!("{}Hello world friend.", short_prefix())); - } - - #[test] - fn prepare_prompt_does_not_double_capitalize_already_uppercase() { - let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); - assert_eq!(out.text, format!("{}HELLO there.", short_prefix())); - } - - #[test] - fn prepare_prompt_handles_non_ascii_first_letter() { - // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce - // mojibake. - let out = prepare_pocket_prompt("дa").expect("non-empty"); - assert!(out.text.contains("Дa.")); - } - - /// REGRESSION GUARD: short prompts must receive *only* whitespace - /// padding — no sacrificial text. A previous revision prepended a - /// `". . "` cold-start absorber and trimmed the rendered audio back out - /// with an amplitude threshold that could eat soft word onsets. If - /// non-whitespace ever reappears in the pad, the synth output will - /// contain audio for text the user never wrote. - #[test] - fn prepare_prompt_pad_is_whitespace_only() { - let out = prepare_pocket_prompt("I'm happy.").expect("non-empty"); - let pad_len = out.text.len() - "I'm happy.".len(); - assert!( - out.text[..pad_len].chars().all(|c| c == ' '), - "short-prompt pad must be spaces only, got {:?}", - &out.text[..pad_len] - ); - assert_eq!(out.text, format!("{}I'm happy.", short_prefix())); - } - - // ── build_generation_extra ─────────────────────────────────────────────── - // - // These tests pin down a behaviour we've now regressed twice on: - // 1) Not padding/punctuating short inputs → 40 s of "monster breathing" - // (pre-773a2a1). - // 2) Setting `frames_after_eos = 1` on long inputs → clipped leading - // audio of multi-clause sentences, e.g. "Yep, I can hear you. …" - // came out as a static burst (the 773a2a1 regression Tyler hit on - // 2026-05-18 ~14:30 UTC). - // - // The contract we enforce going forward: we **only** override - // `max_frames`, and only for ≤4-word inputs. Every other knob is left - // at sherpa-onnx's documented default (notably `frames_after_eos = 3`). - - #[test] - fn build_extra_short_prompt_sets_only_max_frames() { - let prepared = prepare_pocket_prompt("yep").expect("non-empty"); - let extra = build_generation_extra(&prepared).expect("short prompts get extra"); - // Exactly one key — `max_frames` — and nothing else. - assert_eq!(extra.len(), 1, "extra has unexpected keys: {extra:?}"); - assert_eq!( - extra.get("max_frames"), - Some(&serde_json::Value::from(SHORT_PROMPT_MAX_FRAMES)) - ); - assert!( - !extra.contains_key("frames_after_eos"), - "frames_after_eos must never be set — upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT} is what we want" - ); - } - - #[test] - fn build_extra_long_prompt_is_none() { - // ≥5 words: no extras at all. This is the key fix for the "first - // 'yep' in 'Yep, I can hear you. …' is static" regression — we - // were previously forcing `frames_after_eos = 1` on this path. - let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); - assert_eq!( - build_generation_extra(&prepared), - None, - "long prompts must not override any LM knob" - ); - } - - #[test] - fn build_extra_never_lowers_frames_after_eos_for_any_word_count() { - // Sweep a range of prompt lengths and assert the `extra` map (when - // present) never carries a `frames_after_eos` override that's lower - // than the upstream sherpa-onnx default. Implemented as a structural - // check — we just never set the key — but worth a property test in - // case someone reintroduces the override in the future. - let prompts: &[&str] = &[ - "hi", - "hi there", - "yes please", - "one two three four", - "one two three four five", - "a slightly longer reply, hopefully fine", - "This is a multi-clause sentence. It has two parts.", - "really really really really really long prompt with lots of words just to be sure", - ]; - for &p in prompts { - let prepared = prepare_pocket_prompt(p).expect("non-empty"); - if let Some(extra) = build_generation_extra(&prepared) { - if let Some(v) = extra.get("frames_after_eos") { - let n = v.as_i64().expect("frames_after_eos should be int"); - assert!( - n >= SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT as i64, - "prompt {p:?} set frames_after_eos={n}, below upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT}" - ); - } - } - } - } - - #[test] - fn short_prompt_max_frames_is_below_upstream_default() { - // Sanity: the override only ever *lowers* the cap, never raises it. - const { - assert!(SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT); - } - // …and is still large enough for a one-to-four-word reply. At Mimi's - // 12.5 Hz frame rate, 100 frames = 8 s, which is roomy. - const { - assert!(SHORT_PROMPT_MAX_FRAMES >= 50, "would risk truncation"); - } - } -} +pub use buzz_voice_pkg::pocket::*; diff --git a/desktop/src-tauri/src/huddle/state.rs b/desktop/src-tauri/src/huddle/state.rs index 876c2d688b..55382a9880 100644 --- a/desktop/src-tauri/src/huddle/state.rs +++ b/desktop/src-tauri/src/huddle/state.rs @@ -202,8 +202,10 @@ impl HuddleState { /// to invalidate in-flight transcription tasks without losing the generation. pub(crate) fn reset_preserving_generation(&mut self) { let gen = Arc::clone(&self.session_generation); + let tts_enabled = self.tts_enabled; *self = Self::default(); self.session_generation = gen; + self.tts_enabled = tts_enabled; } } @@ -233,3 +235,20 @@ pub fn emit_huddle_state(app: &tauri::AppHandle, state: &HuddleState) { pub struct HuddleJoinInfo { pub ephemeral_channel_id: String, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn teardown_preserves_installation_global_tts_preference() { + let mut state = HuddleState { + tts_enabled: false, + phase: HuddlePhase::Active, + ..HuddleState::default() + }; + state.reset_preserving_generation(); + assert!(!state.tts_enabled); + assert_eq!(state.phase, HuddlePhase::Idle); + } +} diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 63a435cd8e..4bf1cc02f5 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -35,10 +35,11 @@ //! can gate microphone input while the agent is speaking. use std::{ + collections::VecDeque, num::NonZero, path::PathBuf, sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, mpsc::{self, SyncSender}, Arc, Mutex, MutexGuard, PoisonError, }, @@ -46,9 +47,16 @@ use std::{ time::Duration, }; -use super::pocket::{load_text_to_speech, load_voice_style, SAMPLE_RATE, VOICE_FILE_EXT}; +use super::playback_speed::{process_complete_chunk_preserving_lead_in, PlaybackSpeedControl}; +use super::pocket::{ + load_text_to_speech, load_voice_style, SynthesisOutcome, SAMPLE_RATE, VOICE_FILE_EXT, +}; use super::preprocessing::{preprocess_for_tts, split_sentences}; +#[path = "tts_voice_transition.rs"] +mod voice_transition; +use voice_transition::*; + // ── Constants ───────────────────────────────────────────────────────────────── /// Maximum number of queued text items. @@ -122,7 +130,7 @@ const INTER_SENTENCE_SILENCE: f32 = 0.1; #[derive(Debug)] pub struct TtsPipeline { /// Send preprocessed text into the pipeline. - text_tx: SyncSender, + text_tx: SyncSender, /// `true` while the agent is speaking. Shared with the STT pipeline for gating. #[allow(dead_code)] pub tts_active: Arc, @@ -132,67 +140,67 @@ pub struct TtsPipeline { /// Kept alive here so the Arc isn't dropped — the worker holds a clone. #[allow(dead_code)] cancel: Arc, - /// Voice name (e.g. "reference_sample"). Stored for future voice-switching support. - #[allow(dead_code)] - voice: String, + /// Internal cancellation used only for voice changes. Kept separate so a + /// concurrent human barge-in always clears every queued message. + voice_cancel: Arc, + /// Selected manifest voice. The worker reloads only the lightweight style + /// when this changes; the warmed Pocket engine and audio player stay alive. + voice: Arc>, + /// Tags messages so a voice change drops only pre-change queue entries. + voice_generation: Arc, + /// Completed after the worker drains pre-change text and installs the new style. + voice_change_ack: VoiceChangeAck, /// Worker thread handle — taken on drop to join cleanly. thread: Option>, } impl TtsPipeline { - /// Spawn the TTS pipeline thread using the default voice. - /// - /// `model_dir` must contain the Pocket TTS files declared by `huddle::models` - /// (the five ONNX sessions, the two JSON tables, and `.wav`). - /// - /// `tts_active` is set to `true` while audio is playing and `false` when idle. - /// Pass the same `Arc` to the STT pipeline to gate microphone input. + /// Spawn the TTS pipeline thread with a manifest-backed voice name. /// - /// `cancel` is the shared barge-in flag from `HuddleState.tts_cancel`. Pass the - /// same `Arc` to the STT pipeline so both sides reference the same flag for the - /// entire huddle session — no stale references after pipeline restarts. - pub fn new( - model_dir: PathBuf, - tts_active: Arc, - cancel: Arc, - output_device: Option, - ) -> Result { - use super::pocket::DEFAULT_VOICE; - Self::new_with_voice(model_dir, tts_active, cancel, DEFAULT_VOICE, output_device) - } - - /// Spawn the TTS pipeline thread with a specific voice name. Today only the - /// bundled default voice (see `pocket::DEFAULT_VOICE`) is shipped; other - /// names will surface a clear error from `load_voice_style`. + /// `cancel` is shared with STT for barge-in. The same handle survives voice + /// changes so the warmed Pocket engine is retained. pub fn new_with_voice( model_dir: PathBuf, tts_active: Arc, cancel: Arc, voice: &str, output_device: Option, + playback_speed: PlaybackSpeedControl, ) -> Result { - let (text_tx, text_rx) = mpsc::sync_channel::(TEXT_QUEUE_DEPTH); + let (text_tx, text_rx) = mpsc::sync_channel::(TEXT_QUEUE_DEPTH); let shutdown = Arc::new(AtomicBool::new(false)); // cancel is passed in from HuddleState.tts_cancel — shared with STT for barge-in. let shutdown_worker = Arc::clone(&shutdown); let cancel_worker = Arc::clone(&cancel); + let voice_cancel = Arc::new(AtomicBool::new(false)); + let worker_voice_cancel = Arc::clone(&voice_cancel); let tts_active_worker = Arc::clone(&tts_active); - let voice_name = voice.to_string(); + let voice = Arc::new(Mutex::new(voice.to_string())); + let voice_worker = Arc::clone(&voice); + let voice_generation = Arc::new(AtomicU64::new(1)); + let worker_voice_generation = Arc::clone(&voice_generation); + let voice_change_ack = Arc::new(Mutex::new(None)); + let worker_voice_change_ack = Arc::clone(&voice_change_ack); let model_dir_worker = model_dir.clone(); let handle = thread::Builder::new() .name("tts-worker".into()) .spawn(move || { - tts_worker( - model_dir_worker, - voice_name, - text_rx, - tts_active_worker, - shutdown_worker, - cancel_worker, + let config = TtsWorkerConfig { + model_dir: model_dir_worker, + voice_state: ( + voice_worker, + worker_voice_generation, + worker_voice_change_ack, + ), + tts_active: tts_active_worker, + shutdown: shutdown_worker, + cancel_signals: (cancel_worker, worker_voice_cancel), output_device, - ) + playback_speed, + }; + tts_worker(config, text_rx) }) .map_err(|e| format!("failed to spawn tts-worker thread: {e}"))?; @@ -201,7 +209,10 @@ impl TtsPipeline { tts_active, shutdown, cancel, - voice: voice.to_string(), + voice_cancel, + voice, + voice_generation, + voice_change_ack, thread: Some(handle), }) } @@ -211,10 +222,49 @@ impl TtsPipeline { /// Non-blocking. Returns `Err` if the queue is full (bounded at /// `TEXT_QUEUE_DEPTH`) — caller may log and discard. pub fn speak(&self, text: String) -> Result<(), String> { - self.text_tx.try_send(text).map_err(|e| { - eprintln!("buzz-desktop: TTS queue saturated, dropping message: {e}"); - format!("TTS queue full, dropping: {e}") - }) + self.text_tx + .try_send(QueuedText { + generation: self.voice_generation.load(Ordering::Acquire), + text, + }) + .map_err(|e| { + eprintln!("buzz-desktop: TTS queue saturated, dropping message: {e}"); + format!("TTS queue full, dropping: {e}") + }) + } + + /// Clone the bounded queue sender so callers can apply backpressure without + /// holding the huddle mutex. Disabling TTS drops the receiver and unblocks + /// any waiting sender while the shared cancellation flag stops playback. + pub(crate) fn text_sender(&self) -> TtsTextSender { + TtsTextSender { + text_tx: self.text_tx.clone(), + generation: self.voice_generation.load(Ordering::Acquire), + } + } + + /// Select a bundled Pocket voice for subsequent speech. + /// + /// Current playback and queued text are cancelled immediately so content + /// cannot continue in the old voice. The worker keeps its warmed inference + /// engine and reloads only the reference style before the next utterance. + pub fn select_voice(&self, voice: &str) -> Option> { + begin_voice_change( + &self.voice, + &self.voice_generation, + &self.voice_cancel, + &self.voice_change_ack, + voice, + ) + } + + /// Reconcile the voice of a pipeline that has not been published yet. + /// + /// No caller can enqueue text before publication, so raising the shared + /// cancellation flag here would create a race that could discard the first + /// message queued immediately after installation. + pub(crate) fn select_voice_before_publish(&self, voice: &str) { + *self.voice.lock().unwrap_or_else(|error| error.into_inner()) = voice.to_string(); } /// Signal the worker thread to stop. @@ -242,15 +292,28 @@ impl Drop for TtsPipeline { // ── Worker thread ───────────────────────────────────────────────────────────── -fn tts_worker( +struct TtsWorkerConfig { model_dir: PathBuf, - voice_name: String, - text_rx: mpsc::Receiver, + voice_state: WorkerVoiceState, tts_active: Arc, shutdown: Arc, - cancel: Arc, + cancel_signals: WorkerCancelSignals, output_device: Option, -) { + playback_speed: PlaybackSpeedControl, +} + +fn tts_worker(config: TtsWorkerConfig, text_rx: mpsc::Receiver) { + let TtsWorkerConfig { + model_dir, + voice_state, + tts_active, + shutdown, + cancel_signals, + output_device, + playback_speed, + } = config; + let (selected_voice, voice_generation, voice_change_ack) = voice_state; + let (cancel, voice_cancel) = cancel_signals; // ── 1. Initialise TTS engine ────────────────────────────────────────────── let model_dir_str = model_dir.to_string_lossy().to_string(); @@ -261,20 +324,34 @@ fn tts_worker( "buzz-desktop: TTS engine init failed (model_dir={}): {e}. TTS disabled.", model_dir.display() ); - drain_until_shutdown(text_rx, &shutdown); + drain_tts_until_shutdown( + text_rx, + &shutdown, + (&cancel, &voice_cancel), + &voice_change_ack, + ); return; } }; // ── 2. Load voice style ─────────────────────────────────────────────────── + let mut voice_name = selected_voice + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); let voice_path = model_dir.join(format!("{voice_name}.{VOICE_FILE_EXT}")); - let style = match load_voice_style(&voice_path) { + let mut style = match load_voice_style(&voice_path) { Ok(s) => s, Err(e) => { eprintln!( "buzz-desktop: TTS voice style load failed ({voice_name}): {e}. TTS disabled." ); - drain_until_shutdown(text_rx, &shutdown); + drain_tts_until_shutdown( + text_rx, + &shutdown, + (&cancel, &voice_cancel), + &voice_change_ack, + ); return; } }; @@ -307,7 +384,12 @@ fn tts_worker( Ok(h) => h, Err(e) => { eprintln!("buzz-desktop: TTS audio output failed: {e}. TTS disabled."); - drain_until_shutdown(text_rx, &shutdown); + drain_tts_until_shutdown( + text_rx, + &shutdown, + (&cancel, &voice_cancel), + &voice_change_ack, + ); return; } }; @@ -377,6 +459,7 @@ fn tts_worker( let monitor = { let player = Arc::clone(&player); let cancel = Arc::clone(&cancel); + let voice_cancel = Arc::clone(&voice_cancel); let tts_active = Arc::clone(&tts_active); let stop = Arc::clone(&monitor_stop); let player_ops = Arc::clone(&player_ops); @@ -384,12 +467,12 @@ fn tts_worker( .name("tts-barge-in-monitor".into()) .spawn(move || { while !stop.load(Ordering::Acquire) { - if cancel.load(Ordering::Acquire) { + if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) { let _ops = lock_player_ops(&player_ops); // Re-check under the lock: the worker may have // consumed this cancel (and appended fresh audio) // between the load above and the lock acquisition. - if cancel.load(Ordering::Acquire) { + if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) { // clear() pauses the persistent player; play() // un-pauses (see handle_cancel_or_shutdown). // Idempotent — safe to repeat every tick until @@ -423,13 +506,16 @@ fn tts_worker( // idle branch below uses it to decide when to drop `tts_active` and to // arm a fresh lead-in cushion for the next utterance. let mut first_append = true; + let mut deferred_text = VecDeque::new(); loop { + let mut no_current_text = None; if handle_cancel_or_shutdown( - &cancel, + (&cancel, &voice_cancel), &shutdown, &tts_active, - &text_rx, + (&text_rx, &mut deferred_text, &mut no_current_text), + &voice_change_ack, Some((&player, &player_ops)), ) { if shutdown.load(Ordering::Acquire) { @@ -441,28 +527,42 @@ fn tts_worker( continue; } - let raw_text = match text_rx.recv_timeout(RECV_TIMEOUT) { - Ok(t) => t, - Err(mpsc::RecvTimeoutError::Timeout) => { - // Nothing queued. If playback has also finished, the agent - // has gone quiet — release the mic gate and reset the - // lead-in so the next utterance gets a fresh cushion. - if player.empty() && !first_append { - tts_active.store(false, Ordering::Release); - first_append = true; + // Voice changes cancel the old utterance/queue and are observed here, + // before receiving subsequent text. A bad bundled asset falls back to + // Mary without discarding the already-warmed Pocket engine. + let voice_ready = + reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style); + acknowledge_voice_change(&voice_change_ack, &voice_cancel); + if !voice_ready { + continue; + } + + let mut queued_text = Some(match deferred_text.pop_front() { + Some(text) => text, + None => match text_rx.recv_timeout(RECV_TIMEOUT) { + Ok(text) => text, + Err(mpsc::RecvTimeoutError::Timeout) => { + // Nothing queued. If playback has also finished, the agent + // has gone quiet — release the mic gate and reset the + // lead-in so the next utterance gets a fresh cushion. + if player.empty() && !first_append { + tts_active.store(false, Ordering::Release); + first_append = true; + } + continue; } - continue; - } - Err(mpsc::RecvTimeoutError::Disconnected) => break, - }; + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }, + }); // Check cancel again after unblocking — a cancel may have arrived // while we were waiting. if handle_cancel_or_shutdown( - &cancel, + (&cancel, &voice_cancel), &shutdown, &tts_active, - &text_rx, + (&text_rx, &mut deferred_text, &mut queued_text), + &voice_change_ack, Some((&player, &player_ops)), ) { if shutdown.load(Ordering::Acquire) { @@ -471,6 +571,21 @@ fn tts_worker( first_append = true; continue; } + let Some(queued_text) = queued_text else { + continue; + }; + if queued_text.generation < voice_generation.load(Ordering::Acquire) { + continue; + } + let raw_text = queued_text.text; + + // The selected voice can change while this worker is blocked in + // recv_timeout. Reconcile again after receipt so the first message + // queued after an unpublished pipeline is installed cannot use the + // voice captured when construction began. + if !reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style) { + continue; + } // If playback already drained while we were waiting for this item, // the agent is silent — release the mic gate BEFORE preprocessing/ @@ -505,11 +620,13 @@ fn tts_worker( let chunks = group_sentences_into_chunks(&sentences, MAX_CHUNK_CHARS); for chunk in &chunks { + let mut no_current_text = None; if handle_cancel_or_shutdown( - &cancel, + (&cancel, &voice_cancel), &shutdown, &tts_active, - &text_rx, + (&text_rx, &mut deferred_text, &mut no_current_text), + &voice_change_ack, Some((&player, &player_ops)), ) { first_append = true; @@ -521,8 +638,15 @@ fn tts_worker( continue; } - match engine.synth_chunk(text, "en", &style, SYNTH_STEPS) { - Ok(samples) if !samples.is_empty() => { + let synth_cancel = Arc::clone(&cancel); + let synth_voice_cancel = Arc::clone(&voice_cancel); + let synth_shutdown = Arc::clone(&shutdown); + match engine.synth_chunk_interruptible(text, "en", &style, SYNTH_STEPS, move || { + synth_cancel.load(Ordering::Acquire) + || synth_voice_cancel.load(Ordering::Acquire) + || synth_shutdown.load(Ordering::Acquire) + }) { + Ok(SynthesisOutcome::Complete(samples)) if !samples.is_empty() => { let mut audio = clamp_to_full_scale(samples); // Fade-out only — fading-in would attenuate the consonant // onset (see `apply_fade_out` docstring + the @@ -536,6 +660,22 @@ fn tts_worker( // every chunk a quiet device warm-up window. let buf = build_sentence_append_buffer(&mut first_append, audio, silence_buf_len); + let speed = playback_speed.get(); + let buf = match process_complete_chunk_preserving_lead_in( + &buf, + SENTENCE_LEAD_IN_SAMPLES, + speed, + SAMPLE_RATE, + ) { + Ok(processed) => processed, + Err(error) => { + eprintln!( + "buzz-desktop: TTS playback-speed processing failed at {speed:.2}x: \ + {error}; using 1x playback" + ); + buf + } + }; // Check-and-append under `player_ops`, serialized with // the monitor: a barge-in may have arrived during @@ -549,7 +689,7 @@ fn tts_worker( // does the full consume (drain queue, reset lead-in) on // the next iteration. let _ops = lock_player_ops(&player_ops); - if cancel.load(Ordering::Acquire) { + if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) { // Nothing appended; the loop-top consume re-arms // `first_append` (the flag is still set — the worker // is its only consumer). @@ -563,7 +703,11 @@ fn tts_worker( // See crossfire review C3. tts_active.store(true, Ordering::Release); } - Ok(_) => {} + Ok(SynthesisOutcome::Complete(_)) => {} + Ok(SynthesisOutcome::Interrupted) => { + first_append = true; + break; + } Err(e) => { eprintln!("buzz-desktop: TTS synth failed: {e}"); } @@ -582,11 +726,35 @@ fn tts_worker( let _ = handle.join(); } + finish_voice_change_ack(&voice_change_ack); tts_active.store(false, Ordering::Release); } // ── Helpers ─────────────────────────────────────────────────────────────────── +fn drain_tts_until_shutdown( + text_rx: mpsc::Receiver, + shutdown: &AtomicBool, + cancel_signals: CancelSignals<'_>, + voice_change_ack: &VoiceChangeAck, +) { + let (cancel, voice_cancel) = cancel_signals; + loop { + if cancel.swap(false, Ordering::AcqRel) | voice_cancel.swap(false, Ordering::AcqRel) { + while text_rx.try_recv().is_ok() {} + } + acknowledge_voice_change(voice_change_ack, voice_cancel); + if shutdown.load(Ordering::Acquire) { + break; + } + match text_rx.recv_timeout(RECV_TIMEOUT) { + Ok(_) | Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => break, + } + } + finish_voice_change_ack(voice_change_ack); +} + /// Check for cancel or shutdown. Returns `true` if the caller should break/continue. /// On cancel: drains the text queue and clears the cancel flag. /// @@ -595,12 +763,15 @@ fn tts_worker( /// it is serialized with the monitor's stale-branch re-check (see the monitor /// block in `tts_worker`). fn handle_cancel_or_shutdown( - cancel: &AtomicBool, + cancel_signals: CancelSignals<'_>, shutdown: &AtomicBool, tts_active: &AtomicBool, - text_rx: &mpsc::Receiver, + text_state: CancelTextState<'_>, + voice_change_ack: &VoiceChangeAck, player: Option<(&rodio::Player, &Mutex<()>)>, ) -> bool { + let (cancel, voice_cancel) = cancel_signals; + let (text_rx, deferred_text, current_text) = text_state; if shutdown.load(Ordering::Acquire) { if let Some((p, ops)) = player { let _ops = lock_player_ops(ops); @@ -609,7 +780,24 @@ fn handle_cancel_or_shutdown( tts_active.store(false, Ordering::Release); return true; } - if cancel.load(Ordering::Acquire) { + if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) { + // Serialize with begin_voice_change so the generation boundary and + // cancel consumption are observed as one transition. + let pending_voice_change = voice_change_ack + .lock() + .unwrap_or_else(|error| error.into_inner()); + // Consume at the serialization point. A later barge-in remains true + // for the next pass instead of being overwritten after queue cleanup. + let barge_in = cancel.swap(false, Ordering::AcqRel); + voice_cancel.store(false, Ordering::Release); + let preserve_generation = (!barge_in) + .then(|| { + pending_voice_change + .as_ref() + .map(|pending| pending.generation) + }) + .flatten(); + retain_cancelled_text(deferred_text, current_text, text_rx, preserve_generation); if let Some((p, ops)) = player { let _ops = lock_player_ops(ops); // `Player::clear()` removes queued sources AND pauses the player @@ -622,11 +810,6 @@ fn handle_cancel_or_shutdown( // Consume the flag under the lock: once released with // `cancel == false`, the monitor's stale branch no-ops instead // of clearing the fresh post-cancel utterance. - while text_rx.try_recv().is_ok() {} - cancel.store(false, Ordering::Release); - } else { - while text_rx.try_recv().is_ok() {} - cancel.store(false, Ordering::Release); } tts_active.store(false, Ordering::Release); return true; @@ -768,11 +951,14 @@ fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec 0.1), + "speech energy must remain after the fixed device cushion" + ); +} diff --git a/desktop/src-tauri/src/huddle/tts_settings.rs b/desktop/src-tauri/src/huddle/tts_settings.rs new file mode 100644 index 0000000000..b6c006e76b --- /dev/null +++ b/desktop/src-tauri/src/huddle/tts_settings.rs @@ -0,0 +1,837 @@ +//! Installation-global text-to-speech preferences and the local voice registry. +//! +//! Voice keys are backend-qualified (`pocket:mary`, `siri:aaron`) and +//! preferences are ordered. A client resolves the first compatible entry for +//! its one active playback backend. The same [`VoicePreferences`] value can be +//! embedded in installation-global settings or future agent identity without a +//! schema change. Availability is intentionally client-local. + +use std::{ + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +use serde::{Deserialize, Serialize}; +use tauri::{AppHandle, Manager, State}; + +use crate::{app_state::AppState, managed_agents::storage::atomic_write_json_restricted}; + +use super::{models, pocket::DEFAULT_VOICE, HuddlePhase, HuddleState}; + +const SETTINGS_FILE: &str = "tts-settings.json"; +const CURRENT_VERSION: u32 = 1; +const VOICE_CHANGE_ACK_TIMEOUT: Duration = Duration::from_secs(5); +pub const POCKET_BACKEND_ID: &str = "pocket"; +pub const MARY_VOICE_KEY: &str = "pocket:mary"; +pub const MARIUS_VOICE_KEY: &str = "pocket:marius"; + +type VoiceChangeWait = ( + Arc, + tokio::sync::oneshot::Receiver<()>, +); + +const VOICE_AVAILABILITY_BUNDLED: &str = "bundled"; +const VOICE_AVAILABILITY_INSTALLED: &str = "installed"; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct VoiceRegistryEntry { + /// Stable identity, never derived from or merged by the display name. + /// + /// Built-ins use `backend:slug`. Future imports use + /// `pocket:imported:` so two clips with the same + /// editable label remain distinct. + pub key: String, + pub display_name: String, + pub backend: String, + pub backend_name: String, + /// Client-local state: bundled, installed, downloadable, or unavailable. + pub availability: String, + pub fallback_key: Option, + pub reference_file: Option, + pub provenance: VoiceProvenance, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct VoiceProvenance { + pub source: String, + pub content_hash: Option, + pub license: Option, + pub source_url: Option, +} + +/// Ordered, backend-qualified preferences shared by global and agent settings. +/// +/// Unknown but well-formed keys remain persisted because a different client +/// may have that backend installed. Resolution is always local. +pub type VoicePreferences = Vec; + +/// Cross-backend registry for voices known to this client. +/// +/// V1 contains Pocket entries only. Siri, Kokoro, imported voices, and +/// per-agent assignment can add entries or reuse the preference type without +/// changing the registry/settings boundary. +pub fn voice_registry() -> Vec { + vec![ + VoiceRegistryEntry { + key: MARY_VOICE_KEY.to_string(), + display_name: "Mary".to_string(), + backend: POCKET_BACKEND_ID.to_string(), + backend_name: "Pocket TTS".to_string(), + availability: VOICE_AVAILABILITY_BUNDLED.to_string(), + fallback_key: None, + reference_file: Some("reference_sample.wav".to_string()), + provenance: VoiceProvenance { + source: "bundled".to_string(), + content_hash: Some( + "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f" + .to_string(), + ), + license: Some("CC-BY-4.0".to_string()), + source_url: Some("https://datashare.ed.ac.uk/handle/10283/3443".to_string()), + }, + }, + VoiceRegistryEntry { + key: MARIUS_VOICE_KEY.to_string(), + display_name: "Marius".to_string(), + backend: POCKET_BACKEND_ID.to_string(), + backend_name: "Pocket TTS".to_string(), + availability: VOICE_AVAILABILITY_BUNDLED.to_string(), + fallback_key: Some(MARY_VOICE_KEY.to_string()), + reference_file: Some("marius.wav".to_string()), + provenance: VoiceProvenance { + source: "bundled".to_string(), + content_hash: Some( + "076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16" + .to_string(), + ), + license: Some("CC0-1.0".to_string()), + source_url: Some( + "https://huggingface.co/kyutai/tts-voices/blob/323332d33f997de8394f24a193e1a76df720e01a/voice-donations/Selfie.wav" + .to_string(), + ), + }, + }, + ] +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct TtsSettings { + pub version: u32, + pub agent_text_to_speech: bool, + pub voice_preferences: VoicePreferences, +} + +impl Default for TtsSettings { + fn default() -> Self { + Self { + version: CURRENT_VERSION, + agent_text_to_speech: true, + voice_preferences: vec![MARY_VOICE_KEY.to_string()], + } + } +} + +pub fn voice_by_key(key: &str) -> Option { + voice_registry().into_iter().find(|voice| voice.key == key) +} + +fn is_qualified_voice_key(key: &str) -> bool { + key.split_once(':') + .is_some_and(|(backend, voice)| !backend.is_empty() && !voice.is_empty()) +} + +fn is_locally_available(availability: &str) -> bool { + matches!( + availability, + VOICE_AVAILABILITY_BUNDLED | VOICE_AVAILABILITY_INSTALLED + ) +} + +pub fn resolve_voice_for_backend( + preferences: &[String], + backend: &str, +) -> Result { + let registry = voice_registry(); + preferences + .iter() + .filter_map(|key| registry.iter().find(|voice| voice.key == *key)) + .find(|voice| voice.backend == backend && is_locally_available(voice.availability.as_str())) + .or_else(|| { + registry.iter().find(|voice| { + voice.backend == backend + && voice.fallback_key.is_none() + && is_locally_available(voice.availability.as_str()) + }) + }) + .cloned() + .ok_or_else(|| format!("No locally available fallback voice for backend {backend}")) +} + +pub fn pocket_voice_name(preferences: &[String]) -> String { + resolve_voice_for_backend(preferences, POCKET_BACKEND_ID) + .ok() + .and_then(|voice| voice.reference_file) + .and_then(|file| file.strip_suffix(".wav").map(str::to_string)) + .unwrap_or_else(|| DEFAULT_VOICE.to_string()) +} + +pub(crate) fn settings_path(app: &AppHandle) -> Result { + app.path() + .app_data_dir() + .map(|dir| dir.join(SETTINGS_FILE)) + .map_err(|error| format!("could not locate Buzz settings storage: {error}")) +} + +pub(crate) fn load_from_path(path: &Path) -> Result { + if !path.exists() { + return Ok(TtsSettings::default()); + } + let bytes = std::fs::read(path) + .map_err(|error| format!("could not read text-to-speech settings: {error}"))?; + let value: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|error| format!("text-to-speech settings are not valid JSON: {error}"))?; + + // Unversioned settings are incompatible with the V1 schema. Use + // deterministic V1 defaults rather than interpreting ambiguous fields. + if value.get("version").is_none() { + return Ok(TtsSettings::default()); + } + + let version = value + .get("version") + .and_then(serde_json::Value::as_u64) + .ok_or("text-to-speech settings version is invalid")?; + if version > u64::from(CURRENT_VERSION) { + return Err(format!( + "text-to-speech settings version {version} is newer than this Buzz build supports" + )); + } + + // Legacy V1 settings may contain one bare Pocket `voiceId`. Preserve the + // toggle and qualify it into the ordered cross-backend preference schema. + if value.get("voicePreferences").is_none() { + let legacy_voice = value + .get("voiceId") + .or_else(|| value.get("voice_id")) + .and_then(serde_json::Value::as_str) + .unwrap_or("mary"); + let voice_key = if is_qualified_voice_key(legacy_voice) { + legacy_voice.to_string() + } else { + format!("{POCKET_BACKEND_ID}:{legacy_voice}") + }; + return Ok(TtsSettings { + version: CURRENT_VERSION, + agent_text_to_speech: value + .get("agentTextToSpeech") + .and_then(serde_json::Value::as_bool) + .unwrap_or(true), + voice_preferences: vec![voice_key], + }); + } + + let mut settings: TtsSettings = serde_json::from_value(value) + .map_err(|error| format!("text-to-speech settings are invalid: {error}"))?; + settings.version = CURRENT_VERSION; + if settings.voice_preferences.is_empty() + || settings + .voice_preferences + .iter() + .any(|key| !is_qualified_voice_key(key)) + { + settings.voice_preferences = TtsSettings::default().voice_preferences; + } + Ok(settings) +} + +pub(crate) fn save_to_path(path: &Path, settings: &TtsSettings) -> Result<(), String> { + if settings.voice_preferences.is_empty() { + return Err("At least one voice preference is required".to_string()); + } + if let Some(key) = settings + .voice_preferences + .iter() + .find(|key| !is_qualified_voice_key(key)) + { + return Err(format!( + "Voice preference keys must be backend-qualified: {key}" + )); + } + let payload = serde_json::to_vec_pretty(settings) + .map_err(|error| format!("could not encode text-to-speech settings: {error}"))?; + atomic_write_json_restricted(path, &payload) + .map_err(|error| format!("could not save text-to-speech settings: {error}")) +} + +pub fn load_for_app(app: &AppHandle) -> (TtsSettings, Option) { + let result = settings_path(app).and_then(|path| load_from_path(&path)); + match result { + Ok(settings) => (settings, None), + Err(error) => { + eprintln!("buzz-desktop: {error}; preserving the file and using Mary for this session"); + (TtsSettings::default(), Some(error)) + } + } +} + +#[tauri::command] +pub fn get_tts_settings(state: State<'_, AppState>) -> Result { + if let Some(error) = state + .tts_settings_load_error + .lock() + .map_err(|lock_error| format!("text-to-speech settings lock poisoned: {lock_error}"))? + .clone() + { + return Err(format!( + "Voice settings could not be loaded and were left unchanged: {error}" + )); + } + state + .tts_settings + .lock() + .map(|settings| settings.clone()) + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}")) +} + +#[tauri::command] +pub fn list_voice_registry() -> Vec { + voice_registry() +} + +fn ensure_settings_writable(state: &AppState) -> Result<(), String> { + if let Some(error) = state + .tts_settings_load_error + .lock() + .map_err(|lock_error| format!("text-to-speech settings lock poisoned: {lock_error}"))? + .as_ref() + { + return Err(format!( + "Voice settings were not saved because the existing file could not be loaded: {error}" + )); + } + Ok(()) +} + +fn cancel_huddle_speech( + huddle: &mut super::HuddleState, +) -> Option> { + huddle.tts_enabled = false; + huddle + .tts_cancel + .store(true, std::sync::atomic::Ordering::Release); + huddle.tts_pipeline.take() +} + +fn disable_tts_runtime(state: &AppState) -> Result<(), String> { + let old_pipeline = { + let mut huddle = state.huddle()?; + cancel_huddle_speech(&mut huddle) + }; + if let Some(ref pipeline) = old_pipeline { + pipeline.shutdown(); + } + drop(old_pipeline); + state.emit_huddle_state_changed(); + Ok(()) +} + +fn commit_effective_off(state: &AppState) -> Result<(), String> { + state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? + .agent_text_to_speech = false; + Ok(()) +} + +fn enable_tts_runtime(huddle: &mut HuddleState, voice: &str) -> Option { + huddle.tts_enabled = true; + // OFF removes the pipeline. Clear a prior cancellation only when enabling + // a fresh pipeline; an idempotent ON write must not erase a voice + // transition that the existing worker still needs to drain. + if huddle.tts_pipeline.is_none() { + huddle + .tts_cancel + .store(false, std::sync::atomic::Ordering::Release); + } + huddle.tts_pipeline.as_ref().and_then(|pipeline| { + pipeline + .select_voice(voice) + .map(|acknowledged| (Arc::clone(pipeline), acknowledged)) + }) +} + +async fn apply_tts_settings( + settings: TtsSettings, + app: &AppHandle, + state: &AppState, +) -> Result, String> { + if settings.version != CURRENT_VERSION { + return Err(format!( + "Unsupported text-to-speech settings version: {}", + settings.version + )); + } + + // OFF is safety-sensitive: stop current and queued speech before any disk + // I/O, and never resume it merely because persistence fails. + if !settings.agent_text_to_speech { + disable_tts_runtime(state)?; + commit_effective_off(state)?; + } + + ensure_settings_writable(state)?; + save_to_path(&settings_path(app)?, &settings)?; + + *state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? = + settings.clone(); + + let mut voice_change_wait = None; + if settings.agent_text_to_speech { + let (active, voice_change_ack) = { + let mut huddle = state.huddle()?; + let voice_change_ack = + enable_tts_runtime(&mut huddle, &pocket_voice_name(&settings.voice_preferences)); + ( + matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active), + voice_change_ack, + ) + }; + voice_change_wait = voice_change_ack; + if active { + if let Err(error) = super::pipeline::maybe_start_tts_pipeline(state).await { + eprintln!("buzz-desktop: could not hot-start text to speech: {error}"); + } + } + state.emit_huddle_state_changed(); + } + Ok(voice_change_wait) +} + +fn current_settings(state: &AppState) -> Result { + state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}")) + .map(|settings| settings.clone()) +} + +async fn finish_voice_change(voice_change: Option) -> Result<(), String> { + let Some((pipeline, acknowledged)) = voice_change else { + return Ok(()); + }; + wait_for_voice_change_ack(acknowledged, VOICE_CHANGE_ACK_TIMEOUT, || { + pipeline.is_finished() + }) + .await +} + +async fn wait_for_voice_change_ack( + mut acknowledged: tokio::sync::oneshot::Receiver<()>, + timeout: Duration, + mut worker_is_finished: impl FnMut() -> bool, +) -> Result<(), String> { + let deadline = tokio::time::sleep(timeout); + tokio::pin!(deadline); + loop { + tokio::select! { + _ = &mut acknowledged => return Ok(()), + _ = &mut deadline => { + return Err( + "Pocket TTS is still finishing the previous voice. Turn Agent text to speech off and try again." + .to_string(), + ); + } + _ = tokio::time::sleep(Duration::from_millis(25)) => { + if worker_is_finished() { + return Ok(()); + } + } + } + } +} + +/// Compatibility command for the huddle speaker button. It updates the same +/// installation-global preference as Settings; there is no per-huddle override. +#[tauri::command] +pub async fn set_tts_enabled( + enabled: bool, + app: AppHandle, + state: State<'_, AppState>, +) -> Result { + let transition = state.tts_settings_transition.lock().await; + let mut settings = state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? + .clone(); + settings.agent_text_to_speech = enabled; + let voice_change = apply_tts_settings(settings, &app, &state).await?; + drop(transition); + finish_voice_change(voice_change).await?; + current_settings(&state) +} + +fn settings_with_pocket_voice( + mut settings: TtsSettings, + voice_key: &str, +) -> Result { + let voice = voice_by_key(voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?; + if voice.backend != POCKET_BACKEND_ID || !is_locally_available(&voice.availability) { + return Err("The selected Pocket voice is not available on this device".to_string()); + } + let first_pocket_index = settings + .voice_preferences + .iter() + .position(|key| key.starts_with("pocket:")); + settings + .voice_preferences + .retain(|key| !key.starts_with("pocket:")); + let insert_at = first_pocket_index + .unwrap_or(settings.voice_preferences.len()) + .min(settings.voice_preferences.len()); + settings + .voice_preferences + .insert(insert_at, voice_key.to_string()); + Ok(settings) +} + +#[tauri::command] +pub async fn set_pocket_voice( + voice_key: String, + app: AppHandle, + state: State<'_, AppState>, +) -> Result { + let transition = state.tts_settings_transition.lock().await; + let settings = state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? + .clone(); + let settings = settings_with_pocket_voice(settings, &voice_key)?; + let voice_change = apply_tts_settings(settings, &app, &state).await?; + drop(transition); + finish_voice_change(voice_change).await?; + current_settings(&state) +} + +#[tauri::command] +pub async fn preview_pocket_voice( + voice_key: String, + state: State<'_, AppState>, +) -> Result<(), String> { + let voice = voice_by_key(&voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?; + if voice.backend != POCKET_BACKEND_ID { + return Err("Only Pocket voices can be previewed in this build".to_string()); + } + if !models::is_tts_ready() { + return Err("Voice files are still downloading. Try preview again shortly.".to_string()); + } + let model_dir = models::tts_model_dir().ok_or("Pocket voice files are unavailable")?; + let output_device = state + .audio_output_device + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + let playback_speed = state.tts_playback_speed.clone(); + let voice_name = voice + .reference_file + .and_then(|file| file.strip_suffix(".wav").map(str::to_string)) + .ok_or_else(|| format!("Voice {voice_key} has no local Pocket reference file"))?; + tokio::task::spawn_blocking(move || { + let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let pipeline = super::tts::TtsPipeline::new_with_voice( + model_dir, + active.clone(), + cancel, + &voice_name, + output_device, + playback_speed, + )?; + pipeline.speak("Hello! This is how I’ll read agent responses.".to_string())?; + let started = std::time::Instant::now(); + let mut heard_audio = false; + while started.elapsed() < std::time::Duration::from_secs(30) { + let is_active = active.load(std::sync::atomic::Ordering::Acquire); + heard_audio |= is_active; + if heard_audio && !is_active { + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + Err("Voice preview timed out. Check your audio output and try again.".to_string()) + }) + .await + .map_err(|error| format!("Voice preview task failed: {error}"))? +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn stalled_voice_change_returns_an_actionable_error() { + let (_keep_pending, acknowledged) = tokio::sync::oneshot::channel(); + + let error = wait_for_voice_change_ack(acknowledged, Duration::from_millis(1), || false) + .await + .expect_err("stalled worker should time out"); + + assert!(error.contains("Turn Agent text to speech off")); + } + + #[test] + fn idempotent_enable_preserves_an_existing_pipeline_cancel() { + let state = crate::app_state::build_app_state(); + let model_dir = tempfile::tempdir().expect("temp model dir"); + let cancel = state.huddle().expect("huddle state").tts_cancel.clone(); + let pipeline = Arc::new( + super::super::tts::TtsPipeline::new_with_voice( + model_dir.path().to_path_buf(), + Arc::new(std::sync::atomic::AtomicBool::new(false)), + Arc::clone(&cancel), + "reference_sample", + None, + super::super::playback_speed::PlaybackSpeedControl::default(), + ) + .expect("pipeline"), + ); + pipeline.shutdown(); + for _ in 0..100 { + if pipeline.is_finished() { + break; + } + std::thread::sleep(Duration::from_millis(1)); + } + assert!(pipeline.is_finished(), "test pipeline should stop"); + + cancel.store(true, std::sync::atomic::Ordering::Release); + let mut huddle = state.huddle().expect("huddle state"); + huddle.tts_pipeline = Some(pipeline); + + assert!(enable_tts_runtime(&mut huddle, "reference_sample").is_none()); + assert!(cancel.load(std::sync::atomic::Ordering::Acquire)); + } + + #[test] + fn defaults_are_backwards_compatible_and_use_mary() { + assert_eq!( + TtsSettings::default(), + TtsSettings { + version: 1, + agent_text_to_speech: true, + voice_preferences: vec!["pocket:mary".to_string()], + } + ); + } + + #[test] + fn v1_registry_has_exactly_the_two_reviewed_bundled_voices() { + assert_eq!( + voice_registry() + .iter() + .map(|voice| { + ( + voice.key.as_str(), + voice.display_name.as_str(), + voice.reference_file.as_deref(), + ) + }) + .collect::>(), + vec![ + ("pocket:mary", "Mary", Some("reference_sample.wav")), + ("pocket:marius", "Marius", Some("marius.wav")), + ] + ); + } + + #[test] + fn local_backend_resolution_uses_first_compatible_preference() { + let preferences = vec![ + "siri:aaron".to_string(), + MARIUS_VOICE_KEY.to_string(), + MARY_VOICE_KEY.to_string(), + "kokoro:af_heart".to_string(), + ]; + assert_eq!( + resolve_voice_for_backend(&preferences, POCKET_BACKEND_ID) + .expect("Pocket fallback") + .key, + MARIUS_VOICE_KEY + ); + } + + #[test] + fn unsupported_or_missing_preferences_fall_back_to_backend_default() { + let preferences = vec![ + "siri:aaron".to_string(), + "pocket:imported:deadbeef".to_string(), + ]; + assert_eq!( + resolve_voice_for_backend(&preferences, POCKET_BACKEND_ID) + .expect("Pocket fallback") + .key, + MARY_VOICE_KEY + ); + } + + #[test] + fn identity_is_qualified_key_not_display_label() { + assert!(is_qualified_voice_key("pocket:imported:audio-content-hash")); + assert_ne!(MARY_VOICE_KEY, MARIUS_VOICE_KEY); + let mut registry = voice_registry(); + registry[0].display_name = "Jim".to_string(); + registry[1].display_name = "Jim".to_string(); + assert_eq!(registry[0].display_name, registry[1].display_name); + assert_ne!(registry[0].key, registry[1].key); + assert_eq!( + registry + .iter() + .map(|voice| voice.key.as_str()) + .collect::>() + .len(), + registry.len() + ); + } + + #[test] + fn bundled_marius_asset_has_reviewed_size_and_wav_container() { + let bytes = include_bytes!("../../resources/pocket-voices/marius.wav"); + assert_eq!(bytes.len(), 480_044); + assert_eq!(&bytes[0..4], b"RIFF"); + assert_eq!(&bytes[8..12], b"WAVE"); + assert_eq!( + hex::encode(::digest(bytes)), + "076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16" + ); + } + + #[test] + fn migrates_unversioned_experiment_settings_to_v1_defaults() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(SETTINGS_FILE); + std::fs::write(&path, r#"{"voice":"legacy-experiment"}"#).expect("fixture write"); + assert_eq!( + load_from_path(&path).expect("migration"), + TtsSettings::default() + ); + } + + #[test] + fn migrates_bare_pocket_voice_id_to_qualified_preferences() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(SETTINGS_FILE); + std::fs::write( + &path, + r#"{"version":1,"agentTextToSpeech":false,"voiceId":"marius"}"#, + ) + .expect("fixture write"); + assert_eq!( + load_from_path(&path).expect("migration"), + TtsSettings { + version: 1, + agent_text_to_speech: false, + voice_preferences: vec![MARIUS_VOICE_KEY.to_string()], + } + ); + } + + #[test] + fn unknown_qualified_preferences_are_preserved_for_other_clients() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(SETTINGS_FILE); + std::fs::write( + &path, + r#"{"version":1,"agentTextToSpeech":false,"voicePreferences":["siri:aaron","pocket:imported:abc123"]}"#, + ) + .expect("fixture write"); + let settings = load_from_path(&path).expect("load"); + assert!(!settings.agent_text_to_speech); + assert_eq!( + settings.voice_preferences, + vec!["siri:aaron", "pocket:imported:abc123"] + ); + } + + #[test] + fn rejects_future_schema_versions_clearly() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(SETTINGS_FILE); + std::fs::write( + &path, + r#"{"version":99,"agentTextToSpeech":true,"voicePreferences":["pocket:mary"]}"#, + ) + .expect("fixture write"); + assert!(load_from_path(&path) + .expect_err("future version should fail") + .contains("newer than this Buzz build supports")); + } + + #[test] + fn disabling_cancels_runtime_before_persistence_can_fail() { + let mut huddle = super::super::HuddleState { + tts_enabled: true, + ..super::super::HuddleState::default() + }; + assert!(!huddle.tts_cancel.load(std::sync::atomic::Ordering::Acquire)); + assert!(cancel_huddle_speech(&mut huddle).is_none()); + assert!(!huddle.tts_enabled); + assert!(huddle.tts_cancel.load(std::sync::atomic::Ordering::Acquire)); + } + + #[test] + fn pocket_voice_update_preserves_the_latest_toggle_and_other_backends() { + let current = TtsSettings { + agent_text_to_speech: false, + voice_preferences: vec!["siri:aaron".to_string(), MARY_VOICE_KEY.to_string()], + ..TtsSettings::default() + }; + let updated = + settings_with_pocket_voice(current, MARIUS_VOICE_KEY).expect("available voice"); + assert!(!updated.agent_text_to_speech); + assert_eq!( + updated.voice_preferences, + vec!["siri:aaron", MARIUS_VOICE_KEY] + ); + } + + #[test] + fn failed_off_persistence_cannot_be_undone_by_a_later_voice_update() { + let state = crate::app_state::build_app_state(); + commit_effective_off(&state).expect("commit effective OFF state"); + + // This models the next command after the OFF save fails: it must merge + // from effective memory state, not the stale last-persisted ON value. + let current = state.tts_settings.lock().expect("settings").clone(); + let voice_update = + settings_with_pocket_voice(current, MARIUS_VOICE_KEY).expect("available voice"); + assert!(!voice_update.agent_text_to_speech); + } + + #[test] + fn failed_disabled_voice_save_does_not_change_the_remembered_voice() { + let state = crate::app_state::build_app_state(); + state + .tts_settings + .lock() + .expect("settings") + .agent_text_to_speech = false; + let current = state.tts_settings.lock().expect("settings").clone(); + let unsaved = + settings_with_pocket_voice(current, MARIUS_VOICE_KEY).expect("available voice"); + + // This is the only pre-persistence mutation for an OFF candidate. + commit_effective_off(&state).expect("commit effective OFF state"); + let remembered = state.tts_settings.lock().expect("settings").clone(); + assert_eq!(remembered.voice_preferences, vec![MARY_VOICE_KEY]); + assert_eq!(unsaved.voice_preferences, vec![MARIUS_VOICE_KEY]); + } +} diff --git a/desktop/src-tauri/src/huddle/tts_voice_selection_tests.rs b/desktop/src-tauri/src/huddle/tts_voice_selection_tests.rs new file mode 100644 index 0000000000..cdf646a02f --- /dev/null +++ b/desktop/src-tauri/src/huddle/tts_voice_selection_tests.rs @@ -0,0 +1,340 @@ +use super::*; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +#[test] +fn selecting_a_voice_raises_only_the_internal_cancel_and_retains_the_engine_handle() { + let model_dir = tempfile::tempdir().expect("temp model dir"); + let active = Arc::new(AtomicBool::new(false)); + let cancel = Arc::new(AtomicBool::new(false)); + let pipeline = TtsPipeline::new_with_voice( + model_dir.path().to_path_buf(), + active, + Arc::clone(&cancel), + "reference_sample", + None, + PlaybackSpeedControl::default(), + ) + .expect("pipeline handle"); + + let _acknowledged = pipeline.select_voice("marius"); + + assert!(!cancel.load(Ordering::Acquire)); + assert!(pipeline.voice_cancel.load(Ordering::Acquire)); + assert_eq!( + pipeline + .voice + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_str(), + "marius" + ); +} + +#[test] +fn reconciling_an_unpublished_pipeline_does_not_cancel_its_first_message() { + let model_dir = tempfile::tempdir().expect("temp model dir"); + let active = Arc::new(AtomicBool::new(false)); + let cancel = Arc::new(AtomicBool::new(false)); + let pipeline = TtsPipeline::new_with_voice( + model_dir.path().to_path_buf(), + active, + Arc::clone(&cancel), + "reference_sample", + None, + PlaybackSpeedControl::default(), + ) + .expect("pipeline handle"); + + pipeline.select_voice_before_publish("marius"); + + assert!(!cancel.load(Ordering::Acquire)); + assert_eq!( + pipeline + .voice + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_str(), + "marius" + ); +} + +#[test] +fn received_text_reconciles_a_voice_changed_while_the_worker_was_waiting() { + let model_dir = tempfile::tempdir().expect("temp model dir"); + let bundled_voice = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("resources/pocket-voices/marius.wav"); + std::fs::copy( + &bundled_voice, + model_dir.path().join("reference_sample.wav"), + ) + .expect("Mary test voice"); + std::fs::copy(&bundled_voice, model_dir.path().join("marius.wav")).expect("Marius test voice"); + + let selected_voice = Arc::new(std::sync::Mutex::new("reference_sample".to_string())); + let mut style = + load_voice_style(&model_dir.path().join("reference_sample.wav")).expect("initial style"); + let waiting = Arc::new(std::sync::Barrier::new(2)); + let (text_tx, text_rx) = std::sync::mpsc::channel(); + let worker_voice = Arc::clone(&selected_voice); + let worker_waiting = Arc::clone(&waiting); + let worker_model_dir = model_dir.path().to_path_buf(); + let worker = std::thread::spawn(move || { + let mut voice_name = "reference_sample".to_string(); + worker_waiting.wait(); + let text = text_rx.recv().expect("first queued text"); + assert!(reconcile_selected_voice( + &worker_model_dir, + &worker_voice, + &mut voice_name, + &mut style, + )); + (text, voice_name) + }); + + waiting.wait(); + *selected_voice.lock().expect("selected voice") = "marius".to_string(); + text_tx + .send("first message".to_string()) + .expect("queue first message"); + + assert_eq!( + worker.join().expect("worker"), + ("first message".to_string(), "marius".to_string()) + ); +} + +#[test] +fn an_in_hand_post_change_message_survives_cancellation() { + let selected_voice = Arc::new(std::sync::Mutex::new("reference_sample".to_string())); + let voice_generation = AtomicU64::new(1); + let barge_in = AtomicBool::new(false); + let voice_cancel = Arc::new(AtomicBool::new(false)); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (text_tx, text_rx) = std::sync::mpsc::sync_channel(1); + let mut acknowledged = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "marius", + ) + .expect("voice changed"); + assert!(voice_cancel.load(Ordering::Acquire)); + assert!(matches!( + acknowledged.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + acknowledge_voice_change(&voice_change_ack, &voice_cancel); + assert!(matches!( + acknowledged.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + text_tx + .send(QueuedText { + generation: voice_generation.load(Ordering::Acquire), + text: "new message".to_string(), + }) + .expect("new message"); + let mut current_text = Some(text_rx.recv().expect("in-hand new message")); + + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let mut deferred_text = VecDeque::from([ + QueuedText { + generation: 1, + text: "old message".to_string(), + }, + QueuedText { + generation: voice_generation.load(Ordering::Acquire), + text: "later new message".to_string(), + }, + ]); + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + acknowledge_voice_change(&voice_change_ack, &voice_cancel); + acknowledged.blocking_recv().expect("voice change ack"); + + assert_eq!( + deferred_text + .pop_front() + .expect("preserved post-change message") + .text, + "new message" + ); + assert_eq!( + deferred_text + .pop_front() + .expect("later post-change message") + .text, + "later new message" + ); + assert!(text_rx.try_recv().is_err()); +} + +#[test] +fn superseding_voice_change_removes_earlier_deferred_messages() { + let selected_voice = std::sync::Mutex::new("reference_sample".to_string()); + let voice_generation = AtomicU64::new(1); + let barge_in = AtomicBool::new(false); + let voice_cancel = AtomicBool::new(false); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (_text_tx, text_rx) = std::sync::mpsc::channel(); + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let mut deferred_text = VecDeque::new(); + let mut current_text = None; + + let first = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "marius", + ) + .expect("first voice change"); + deferred_text.push_back(QueuedText { + generation: voice_generation.load(Ordering::Acquire), + text: "message for Marius".to_string(), + }); + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + acknowledge_voice_change(&voice_change_ack, &voice_cancel); + first.blocking_recv().expect("first acknowledgement"); + + let _second = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "reference_sample", + ) + .expect("second voice change"); + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + + assert!(deferred_text.is_empty()); +} + +#[test] +fn barge_in_clears_deferred_voice_change_messages() { + let barge_in = AtomicBool::new(true); + let voice_cancel = AtomicBool::new(false); + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (_text_tx, text_rx) = std::sync::mpsc::channel(); + let mut deferred_text = VecDeque::from([QueuedText { + generation: 2, + text: "deferred message".to_string(), + }]); + let mut current_text = None; + + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + + assert!(deferred_text.is_empty()); +} + +#[test] +fn barge_in_during_a_voice_change_clears_post_change_messages() { + let selected_voice = std::sync::Mutex::new("reference_sample".to_string()); + let voice_generation = AtomicU64::new(1); + let barge_in = AtomicBool::new(false); + let voice_cancel = AtomicBool::new(false); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (_text_tx, text_rx) = std::sync::mpsc::channel(); + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let mut deferred_text = VecDeque::new(); + let mut current_text = None; + + let _acknowledged = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "marius", + ) + .expect("voice change"); + deferred_text.push_back(QueuedText { + generation: voice_generation.load(Ordering::Acquire), + text: "post-change message".to_string(), + }); + barge_in.store(true, Ordering::Release); + + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + assert!(deferred_text.is_empty()); +} + +#[test] +fn a_sender_captured_before_voice_change_is_stale_even_if_it_sends_after_drain() { + let selected_voice = std::sync::Mutex::new("reference_sample".to_string()); + let voice_generation = Arc::new(AtomicU64::new(1)); + let barge_in = AtomicBool::new(false); + let voice_cancel = AtomicBool::new(false); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (text_tx, text_rx) = std::sync::mpsc::sync_channel(1); + let old_sender = TtsTextSender { + text_tx, + generation: voice_generation.load(Ordering::Acquire), + }; + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let mut deferred_text = VecDeque::new(); + let mut current_text = None; + + let _acknowledged = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "marius", + ) + .expect("voice change"); + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + old_sender + .send("late old message".to_string()) + .expect("late send"); + let late = text_rx.recv().expect("late queued text"); + + assert!(late.generation < voice_generation.load(Ordering::Acquire)); +} diff --git a/desktop/src-tauri/src/huddle/tts_voice_transition.rs b/desktop/src-tauri/src/huddle/tts_voice_transition.rs new file mode 100644 index 0000000000..1839db0ee7 --- /dev/null +++ b/desktop/src-tauri/src/huddle/tts_voice_transition.rs @@ -0,0 +1,175 @@ +use std::{ + collections::VecDeque, + path::Path, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc::{self, SyncSender}, + Arc, Mutex, + }, +}; + +use crate::huddle::pocket::{load_voice_style, VoiceStyle, DEFAULT_VOICE, VOICE_FILE_EXT}; + +#[derive(Debug)] +pub(super) struct PendingVoiceChange { + pub(super) generation: u64, + acknowledged: tokio::sync::oneshot::Sender<()>, +} + +pub(super) type VoiceChangeAck = Arc>>; +pub(super) type WorkerVoiceState = (Arc>, Arc, VoiceChangeAck); +pub(super) type WorkerCancelSignals = (Arc, Arc); +pub(super) type CancelTextState<'a> = ( + &'a mpsc::Receiver, + &'a mut VecDeque, + &'a mut Option, +); +pub(super) type CancelSignals<'a> = (&'a AtomicBool, &'a AtomicBool); + +#[derive(Debug)] +pub(super) struct QueuedText { + pub(super) generation: u64, + pub(super) text: String, +} + +#[derive(Clone, Debug)] +pub(crate) struct TtsTextSender { + pub(super) text_tx: SyncSender, + pub(super) generation: u64, +} + +impl TtsTextSender { + pub(crate) fn send(&self, text: String) -> Result<(), String> { + self.text_tx + .send(QueuedText { + generation: self.generation, + text, + }) + .map_err(|error| error.to_string()) + } +} + +pub(super) fn begin_voice_change( + selected_voice: &Mutex, + voice_generation: &AtomicU64, + voice_cancel: &AtomicBool, + voice_change_ack: &VoiceChangeAck, + voice: &str, +) -> Option> { + let mut pending_ack = voice_change_ack + .lock() + .unwrap_or_else(|error| error.into_inner()); + let mut selected = selected_voice + .lock() + .unwrap_or_else(|error| error.into_inner()); + if selected.as_str() == voice { + return None; + } + + let (sender, receiver) = tokio::sync::oneshot::channel(); + voice_cancel.store(true, Ordering::Release); + let generation = voice_generation.fetch_add(1, Ordering::AcqRel) + 1; + if let Some(superseded) = pending_ack.replace(PendingVoiceChange { + generation, + acknowledged: sender, + }) { + let _ = superseded.acknowledged.send(()); + } + *selected = voice.to_string(); + Some(receiver) +} + +pub(super) fn acknowledge_voice_change( + voice_change_ack: &VoiceChangeAck, + voice_cancel: &AtomicBool, +) { + let mut pending_ack = voice_change_ack + .lock() + .unwrap_or_else(|error| error.into_inner()); + if voice_cancel.load(Ordering::Acquire) { + return; + } + if let Some(pending) = pending_ack.take() { + let _ = pending.acknowledged.send(()); + } +} + +pub(super) fn finish_voice_change_ack(voice_change_ack: &VoiceChangeAck) { + if let Some(pending) = voice_change_ack + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + let _ = pending.acknowledged.send(()); + } +} + +pub(super) fn reconcile_selected_voice( + model_dir: &Path, + selected_voice: &Mutex, + voice_name: &mut String, + style: &mut VoiceStyle, +) -> bool { + let requested_voice = selected_voice + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + if requested_voice == *voice_name { + return true; + } + + let requested_path = model_dir.join(format!("{requested_voice}.{VOICE_FILE_EXT}")); + match load_voice_style(&requested_path) { + Ok(requested_style) => { + *style = requested_style; + *voice_name = requested_voice; + true + } + Err(error) => { + eprintln!( + "buzz-desktop: Pocket voice {requested_voice} is unavailable ({error}); falling back to Mary" + ); + let fallback_path = model_dir.join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}")); + match load_voice_style(&fallback_path) { + Ok(fallback_style) => { + *style = fallback_style; + *voice_name = DEFAULT_VOICE.to_string(); + *selected_voice + .lock() + .unwrap_or_else(|lock_error| lock_error.into_inner()) = + DEFAULT_VOICE.to_string(); + true + } + Err(fallback_error) => { + eprintln!("buzz-desktop: Mary voice fallback is unavailable: {fallback_error}"); + false + } + } + } + } +} + +pub(super) fn retain_cancelled_text( + deferred_text: &mut VecDeque, + current_text: &mut Option, + text_rx: &mpsc::Receiver, + preserve_generation: Option, +) { + if let Some(generation) = preserve_generation { + deferred_text.retain(|text| text.generation >= generation); + if let Some(text) = current_text.take() { + if text.generation >= generation { + deferred_text.push_front(text); + } + } + while let Ok(text) = text_rx.try_recv() { + if text.generation >= generation { + deferred_text.push_back(text); + } + } + } else { + deferred_text.clear(); + current_text.take(); + while text_rx.try_recv().is_ok() {} + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 5346791ccf..1d07d047aa 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -39,6 +39,7 @@ use deep_link::{ use huddle::audio_output::{ get_audio_output_device, list_audio_output_devices, set_audio_output_device, }; +use huddle::playback_speed::{get_tts_playback_speed, load_playback_speed, set_tts_playback_speed}; use huddle::reconnect::reconnect_huddle_audio; use huddle::{ add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models, @@ -400,6 +401,9 @@ pub fn run() { // that will be lost on restart, as that silently breaks channel // memberships, DMs, and relay identity. let state = app_handle.state::(); + if let Err(error) = load_playback_speed(&app_handle, &state.tts_playback_speed) { + eprintln!("buzz-desktop: failed to load TTS playback speed; using 1x: {error}"); + } if let Err(e) = resolve_persisted_identity(&app_handle, &state) { eprintln!("buzz-desktop: fatal: identity resolution failed: {e}"); std::process::exit(1); @@ -460,6 +464,18 @@ pub fn run() { *guard = Some(app_handle.clone()); } + let (tts_settings, tts_settings_load_error) = + huddle::tts_settings::load_for_app(&app_handle); + if let Ok(mut guard) = state.tts_settings.lock() { + *guard = tts_settings.clone(); + } + if let Ok(mut guard) = state.tts_settings_load_error.lock() { + *guard = tts_settings_load_error; + } + if let Ok(mut huddle) = state.huddle_state.lock() { + huddle.tts_enabled = tts_settings.agent_text_to_speech; + } + // Bring up the runtime-owned shared-compute coordinator before // saved agents are restored. Its lifetime is tied to the app, not // a UI mount; it publishes discovery and reconciles membership for @@ -877,6 +893,10 @@ pub fn run() { download_voice_models, get_model_status, set_tts_enabled, + huddle::tts_settings::get_tts_settings, + huddle::tts_settings::list_voice_registry, + huddle::tts_settings::set_pocket_voice, + huddle::tts_settings::preview_pocket_voice, speak_agent_message, add_agent_to_huddle, check_pipeline_hotstart, @@ -888,6 +908,8 @@ pub fn run() { list_audio_output_devices, set_audio_output_device, get_audio_output_device, + get_tts_playback_speed, + set_tts_playback_speed, start_pairing, confirm_pairing_sas, cancel_pairing, diff --git a/desktop/src/features/huddle/lib/ttsLiveMessages.test.mjs b/desktop/src/features/huddle/lib/ttsLiveMessages.test.mjs new file mode 100644 index 0000000000..088a6d6ebf --- /dev/null +++ b/desktop/src/features/huddle/lib/ttsLiveMessages.test.mjs @@ -0,0 +1,154 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + createInitialMembershipGate, + createLatestStateGate, + createOrderedSpeaker, + speakableAgentText, +} from "./ttsLiveMessages.ts"; + +const agents = new Set(["agent"]); +const base = { + id: "1", + kind: 9, + pubkey: "agent", + content: "Hello there", + tags: [], +}; + +test("speaks only new agent-authored text message events", () => { + assert.equal(speakableAgentText(base, agents, "human"), "Hello there"); + assert.equal( + speakableAgentText({ ...base, kind: 7 }, agents, "human"), + null, + "reactions and other event kinds are excluded", + ); + assert.equal( + speakableAgentText({ ...base, kind: 10 }, agents, "human"), + null, + "edits and status events are excluded", + ); + assert.equal( + speakableAgentText({ ...base, pubkey: "human" }, agents, "human"), + null, + "human-authored messages are excluded", + ); + assert.equal( + speakableAgentText({ ...base, content: " " }, agents, "human"), + null, + "empty and non-text content are excluded", + ); + assert.equal( + speakableAgentText( + { ...base, content: "[System] tool started" }, + agents, + "human", + ), + null, + "legacy system rows are excluded", + ); +}); + +test("strips attachment markup and skips attachment-only events", () => { + const url = "https://cdn.example/voice.png"; + const tags = [["imeta", `url ${url}`, "m image/png"]]; + assert.equal( + speakableAgentText( + { ...base, content: `![image](${url})`, tags }, + agents, + "human", + ), + null, + ); + assert.equal( + speakableAgentText( + { ...base, content: `Here is the diagram.\n\n![image](${url})`, tags }, + agents, + "human", + ), + "Here is the diagram.", + ); + assert.equal( + speakableAgentText( + { ...base, content: `||\n![image](${url})\n||`, tags }, + agents, + "human", + ), + null, + ); +}); + +test("queues agent messages in live thread arrival order", async () => { + const spoken = []; + let releaseFirst; + const firstBlocked = new Promise((resolve) => { + releaseFirst = resolve; + }); + const speaker = createOrderedSpeaker(async (text) => { + if (text === "first") await firstBlocked; + spoken.push(text); + }, assert.fail); + + speaker.enqueue("first"); + speaker.enqueue("second"); + await Promise.resolve(); + assert.deepEqual(spoken, []); + releaseFirst(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.deepEqual(spoken, ["first", "second"]); +}); + +test("disabling cancels queued speech and rejects new messages until enabled", async () => { + const invoked = []; + let releaseFirst; + const firstBlocked = new Promise((resolve) => { + releaseFirst = resolve; + }); + const speaker = createOrderedSpeaker(async (text) => { + invoked.push(text); + if (text === "first") await firstBlocked; + }, assert.fail); + + speaker.enqueue("first"); + speaker.enqueue("queued-before-off"); + await Promise.resolve(); + speaker.setEnabled(false); + speaker.enqueue("while-off"); + releaseFirst(); + await new Promise((resolve) => setTimeout(resolve, 0)); + speaker.setEnabled(true); + speaker.enqueue("after-on"); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.deepEqual(invoked, ["first", "after-on"]); +}); + +test("a live TTS state event supersedes a delayed bootstrap result", () => { + const applied = []; + const gate = createLatestStateGate((enabled) => applied.push(enabled)); + const applyBootstrap = gate.beginSnapshot(); + + gate.applyEvent(false); + applyBootstrap(true); + + assert.deepEqual(applied, [false]); +}); + +test("buffers initial live events until membership resolves in order", () => { + const delivered = []; + const gate = createInitialMembershipGate((event) => delivered.push(event)); + gate.push("first"); + gate.push("second"); + assert.deepEqual(delivered, []); + gate.succeed(); + gate.push("third"); + assert.deepEqual(delivered, ["first", "second", "third"]); +}); + +test("drops the initial buffer fail-closed when membership lookup fails", () => { + const delivered = []; + const gate = createInitialMembershipGate((event) => delivered.push(event)); + gate.push("unverified"); + gate.fail(); + assert.deepEqual(delivered, []); +}); diff --git a/desktop/src/features/huddle/lib/ttsLiveMessages.ts b/desktop/src/features/huddle/lib/ttsLiveMessages.ts new file mode 100644 index 0000000000..1e1aeb8b3f --- /dev/null +++ b/desktop/src/features/huddle/lib/ttsLiveMessages.ts @@ -0,0 +1,124 @@ +export type LiveTtsEvent = { + id: string; + kind: number; + pubkey: string; + content: string; + tags: string[][]; +}; + +function textWithoutAttachments(event: LiveTtsEvent): string { + const urls = new Set( + event.tags + .filter((tag) => tag[0] === "imeta") + .flatMap((tag) => + tag + .slice(1) + .filter((field) => field.startsWith("url ")) + .map((field) => field.slice(4)), + ), + ); + if (urls.size === 0) return event.content; + const withoutMedia = event.content + .split("\n") + .filter( + (line) => !Array.from(urls).some((url) => line.includes(`](${url})`)), + ) + .join("\n"); + return withoutMedia.replace( + /(^|\n)\s*\|\|\s*\n(?:\s*\n)*\s*\|\|\s*(?=\n|$)/gu, + "$1", + ); +} + +export function speakableAgentText( + event: LiveTtsEvent, + agentPubkeys: ReadonlySet, + selfPubkey: string | null, +): string | null { + if (event.kind !== 9) return null; + if (!agentPubkeys.has(event.pubkey)) return null; + if (event.pubkey === selfPubkey) return null; + const content = textWithoutAttachments(event).trim(); + if (content.length <= 1) return null; + if (content.startsWith("[System]")) return null; + return content; +} + +/** + * Serialize native speak calls so live messages enter the bounded Pocket queue + * in thread arrival order even when the bridge resolves calls asynchronously. + */ +export function createOrderedSpeaker( + speak: (text: string) => Promise, + onError: (error: unknown) => void, +): { + enqueue: (text: string) => void; + setEnabled: (enabled: boolean) => void; +} { + let tail = Promise.resolve(); + let enabled = true; + let generation = 0; + return { + enqueue(text) { + if (!enabled) return; + const queuedGeneration = generation; + tail = tail + .then(() => { + if (!enabled || generation !== queuedGeneration) return; + return speak(text); + }) + .catch(onError); + }, + setEnabled(nextEnabled) { + if (!nextEnabled) generation += 1; + enabled = nextEnabled; + }, + }; +} + +/** Ensure a delayed bootstrap snapshot cannot overwrite a newer live event. */ +export function createLatestStateGate(apply: (value: T) => void): { + applyEvent: (value: T) => void; + beginSnapshot: () => (value: T) => void; +} { + let revision = 0; + return { + applyEvent(value) { + revision += 1; + apply(value); + }, + beginSnapshot() { + const snapshotRevision = revision; + return (value) => { + if (revision === snapshotRevision) apply(value); + }; + }, + }; +} + +/** Hold live events until the first authoritative agent-membership lookup. */ +export function createInitialMembershipGate(deliver: (event: T) => void): { + push: (event: T) => void; + succeed: () => void; + fail: () => void; +} { + let settled = false; + let pending: T[] = []; + return { + push(event) { + if (settled) deliver(event); + else pending.push(event); + }, + succeed() { + if (settled) return; + settled = true; + const buffered = pending; + pending = []; + for (const event of buffered) deliver(event); + }, + fail() { + settled = true; + pending = []; + }, + }; +} diff --git a/desktop/src/features/huddle/lib/useTtsSubscription.ts b/desktop/src/features/huddle/lib/useTtsSubscription.ts index d534fc0166..8c6d8f67b5 100644 --- a/desktop/src/features/huddle/lib/useTtsSubscription.ts +++ b/desktop/src/features/huddle/lib/useTtsSubscription.ts @@ -1,7 +1,15 @@ import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; import * as React from "react"; +import { buildHuddleTtsLiveFilter } from "@/shared/api/relayChannelFilters"; import { relayClient } from "@/shared/api/relayClient"; +import { + createInitialMembershipGate, + createLatestStateGate, + createOrderedSpeaker, + speakableAgentText, +} from "./ttsLiveMessages"; const AGENT_PUBKEY_REFRESH_INTERVAL_MS = 30_000; @@ -20,6 +28,7 @@ export function useTtsSubscription( let disposed = false; let cleanup: (() => void) | null = null; + let unlistenHuddleState: (() => void) | null = null; // ── Agent identity (authoritative, fail-closed) ─────────────────────── // @@ -33,66 +42,115 @@ export function useTtsSubscription( let agentsLoaded = false; const agentPubkeys = new Set(); - async function loadAgentPubkeys() { + const speakInOrder = createOrderedSpeaker( + async (text) => { + if (!disposed) { + await invoke("speak_agent_message", { text }); + } + }, + (err) => { + console.warn("[huddle] TTS speak failed:", err); + }, + ); + + const deliver = (event: Parameters[0]) => { + if (!agentsLoaded || disposed) return; + const text = speakableAgentText( + event, + agentPubkeys, + selfPubkeyRef.current, + ); + if (text) speakInOrder.enqueue(text); + }; + const initialMembershipGate = createInitialMembershipGate(deliver); + + async function loadAgentPubkeys(initial = false) { try { const pubkeys = await invoke("get_huddle_agent_pubkeys"); + if (disposed) return; agentPubkeys.clear(); for (const pk of pubkeys) agentPubkeys.add(pk); agentsLoaded = true; + if (initial) { + initialMembershipGate.succeed(); + } } catch (e) { // Fail-closed on ALL failures, including refresh after prior success. // Clear the set and mark as not loaded — TTS goes mute until the // next successful refresh. Stale membership must never authorize speech. agentPubkeys.clear(); agentsLoaded = false; + if (initial) { + initialMembershipGate.fail(); + } console.error("[huddle] Failed to load agent pubkeys:", e); } } // Initial load + periodic refresh (catches mid-huddle agent additions). - void loadAgentPubkeys(); + void loadAgentPubkeys(true); const agentRefreshId = window.setInterval(() => { void loadAgentPubkeys(); }, AGENT_PUBKEY_REFRESH_INTERVAL_MS); + // Install the state listener before requesting a snapshot. If a newer + // event arrives while IPC is pending, it supersedes the stale snapshot. + const ttsStateGate = createLatestStateGate<{ tts_enabled: boolean }>( + (state) => { + if (!disposed) speakInOrder.setEnabled(state.tts_enabled); + }, + ); + void listen<{ tts_enabled: boolean }>("huddle-state-changed", (event) => { + if (!disposed) ttsStateGate.applyEvent(event.payload); + }) + .then((unlisten) => { + if (disposed) { + unlisten(); + return; + } + unlistenHuddleState = unlisten; + const applyBootstrap = ttsStateGate.beginSnapshot(); + void invoke<{ tts_enabled: boolean }>("get_huddle_state") + .then((state) => { + if (!disposed) applyBootstrap(state); + }) + .catch((err) => { + if (!disposed) applyBootstrap({ tts_enabled: false }); + console.warn("[huddle] Failed to load TTS state:", err); + }); + }) + .catch((err) => { + speakInOrder.setEnabled(false); + console.warn("[huddle] Failed to listen for TTS state:", err); + }); + // ── Live-only subscription ─────────────────────────────────────────── - // subscribeToChannelLive uses `since: now` — the relay never sends - // historical backlog. Every event delivered is a live message. + // A kind:9, limit:0 subscription receives future fan-out while the relay + // returns no stored rows, including pre-join rows from the current second. // Event-ID dedup handles reconnect replay (same event arriving twice). const seenEventIds = new Set(); const seenOrder: string[] = []; const MAX_SEEN_EVENTS = 5000; - relayClient - .subscribeToChannelLive(ephemeralChannelId, (event) => { - if (disposed) return; - // Defense-in-depth: subscription already filters to kind:9 only. - if (event.kind !== 9) return; - - // Dedup by event ID (covers reconnect replay). - if (seenEventIds.has(event.id)) return; - seenEventIds.add(event.id); - seenOrder.push(event.id); - if (seenOrder.length > MAX_SEEN_EVENTS) { - const oldest = seenOrder.shift(); - if (oldest !== undefined) seenEventIds.delete(oldest); - } + .subscribeLive( + buildHuddleTtsLiveFilter(ephemeralChannelId), + (event) => { + if (disposed) return; + // Dedup by event ID (covers reconnect replay). + if (seenEventIds.has(event.id)) return; + seenEventIds.add(event.id); + seenOrder.push(event.id); + if (seenOrder.length > MAX_SEEN_EVENTS) { + const oldest = seenOrder.shift(); + if (oldest !== undefined) seenEventIds.delete(oldest); + } - // Fail-closed: don't speak until agent list is loaded. - if (!agentsLoaded) return; - // Only speak agent messages — skip human STT transcripts. - if (!agentPubkeys.has(event.pubkey)) return; - if (event.pubkey === selfPubkeyRef.current) return; - if (event.content.trim().length <= 1) return; - // Legacy: skip [System]-prefixed messages from before kind:48106. - if (event.content.startsWith("[System]")) return; - invoke("speak_agent_message", { text: event.content }).catch((err) => { - console.warn( - "[huddle] TTS speak failed (backpressure or pipeline unavailable):", - err, - ); - }); - }) + // Preserve arrival order while the initial authoritative membership + // lookup is pending. A failed lookup clears this buffer fail-closed. + initialMembershipGate.push(event); + }, + { replayMissedHistory: true }, + ) .then((dispose) => { if (disposed) { void dispose(); @@ -106,7 +164,9 @@ export function useTtsSubscription( return () => { disposed = true; + speakInOrder.setEnabled(false); cleanup?.(); + unlistenHuddleState?.(); window.clearInterval(agentRefreshId); }; }, [ephemeralChannelId, selfPubkeyRef]); diff --git a/desktop/src/features/settings/ui/SettingsPanels.tsx b/desktop/src/features/settings/ui/SettingsPanels.tsx index 156be00b72..27a0674f80 100644 --- a/desktop/src/features/settings/ui/SettingsPanels.tsx +++ b/desktop/src/features/settings/ui/SettingsPanels.tsx @@ -21,6 +21,7 @@ import { SunMoon, Ticket, UserRound, + Volume2, type LucideIcon, } from "lucide-react"; import type { @@ -84,10 +85,12 @@ import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; import { ProfileSettingsCard } from "./ProfileSettingsCard"; import { UpdateChecker } from "../UpdateChecker"; import { SettingsSectionHeader } from "./SettingsSectionHeader"; +import { VoiceSettingsCard } from "./VoiceSettingsCard"; export type SettingsSection = | "profile" | "notifications" + | "voice" | "experimental" | "agents" | "channel-templates" @@ -107,6 +110,7 @@ export const DEFAULT_SETTINGS_SECTION: SettingsSection = "profile"; const SETTINGS_SECTION_VALUES: readonly SettingsSection[] = [ "profile", "notifications", + "voice", "experimental", "agents", "channel-templates", @@ -168,6 +172,11 @@ export const settingsSections: SettingsSectionDescriptor[] = [ label: "Notifications", icon: BellRing, }, + { + value: "voice", + label: "Voice", + icon: Volume2, + }, { value: "experimental", label: "Experiments", @@ -808,6 +817,8 @@ export function renderSettingsSection( onSetSoundForSlot={props.onSetSoundForSlot} /> ); + case "voice": + return ; case "experimental": return ; case "agents": diff --git a/desktop/src/features/settings/ui/SettingsView.tsx b/desktop/src/features/settings/ui/SettingsView.tsx index 20389b420a..8613880571 100644 --- a/desktop/src/features/settings/ui/SettingsView.tsx +++ b/desktop/src/features/settings/ui/SettingsView.tsx @@ -58,6 +58,7 @@ const settingsNavGroups: Array<{ "profile", "appearance", "notifications", + "voice", "shortcuts", "custom-emoji", "local-archive", diff --git a/desktop/src/features/settings/ui/SpeechPlaybackSettings.tsx b/desktop/src/features/settings/ui/SpeechPlaybackSettings.tsx new file mode 100644 index 0000000000..dc5fb58700 --- /dev/null +++ b/desktop/src/features/settings/ui/SpeechPlaybackSettings.tsx @@ -0,0 +1,120 @@ +import { useEffect, useRef, useState } from "react"; +import { getTtsPlaybackSpeed, setTtsPlaybackSpeed } from "@/shared/api/tauri"; +import { Button } from "@/shared/ui/button"; +import { + applyLoadedPlaybackSpeed, + commitPlaybackSpeed, +} from "./playbackSpeedPersistence"; +import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; + +const DEFAULT_SPEED = 1; +const MIN_SPEED = 0.75; +const MAX_SPEED = 1.5; +const SPEED_STEP = 0.05; + +export function SpeechPlaybackSettings() { + const [speed, setSpeed] = useState(DEFAULT_SPEED); + const [error, setError] = useState(null); + const [loaded, setLoaded] = useState(false); + const confirmedSpeed = useRef(DEFAULT_SPEED); + const desiredSpeed = useRef(DEFAULT_SPEED); + const flushPromise = useRef | null>(null); + const hasLocalIntent = useRef(false); + + useEffect(() => { + let active = true; + getTtsPlaybackSpeed() + .then((savedSpeed) => { + if (active) { + applyLoadedPlaybackSpeed( + { + confirmedSpeed, + desiredSpeed, + flushPromise, + hasLocalIntent, + }, + { setSpeed }, + savedSpeed, + ); + } + }) + .catch((cause) => { + if (active) setError(String(cause)); + }) + .finally(() => { + if (active) setLoaded(true); + }); + return () => { + active = false; + }; + }, []); + + const commitSpeed = (nextSpeed: number) => { + commitPlaybackSpeed( + { confirmedSpeed, desiredSpeed, flushPromise, hasLocalIntent }, + { persist: setTtsPlaybackSpeed, setError, setSpeed }, + nextSpeed, + ); + }; + + return ( + + +
+
+
+ +

+ Changes generated speech playback without changing voice pitch. +

+
+
+ + {speed.toFixed(2)}x + + +
+
+ commitSpeed(Number(event.currentTarget.value))} + onChange={(event) => setSpeed(Number(event.currentTarget.value))} + onKeyUp={(event) => commitSpeed(Number(event.currentTarget.value))} + onPointerUp={(event) => + commitSpeed(Number(event.currentTarget.value)) + } + step={SPEED_STEP} + type="range" + value={speed} + /> + {error ? ( +

+ Could not save playback speed: {error} +

+ ) : null} +
+ + + ); +} diff --git a/desktop/src/features/settings/ui/VoiceSettingsCard.tsx b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx new file mode 100644 index 0000000000..0a536d03f8 --- /dev/null +++ b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx @@ -0,0 +1,255 @@ +import * as React from "react"; +import { ChevronDown, Play, Volume2 } from "lucide-react"; + +import { invokeTauri } from "@/shared/api/tauri"; +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuTrigger, +} from "@/shared/ui/dropdown-menu"; +import { Switch } from "@/shared/ui/switch"; +import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; +import { SettingsSectionHeader } from "./SettingsSectionHeader"; +import { SpeechPlaybackSettings } from "./SpeechPlaybackSettings"; +import { + selectedVoiceForBackend, + type VoiceRegistryEntry, + voiceOptionLabel, + voicesForBackend, +} from "./voiceSettingsLogic"; + +export type TtsSettings = { + version: number; + agentTextToSpeech: boolean; + voicePreferences: string[]; +}; + +export function VoiceSettingsCard() { + const [settings, setSettings] = React.useState(null); + const [registry, setRegistry] = React.useState([]); + const [busy, setBusy] = React.useState(false); + const [previewing, setPreviewing] = React.useState(false); + const [error, setError] = React.useState(null); + + React.useEffect(() => { + let disposed = false; + Promise.all([ + invokeTauri("get_tts_settings"), + invokeTauri("list_voice_registry"), + ]) + .then(([nextSettings, nextRegistry]) => { + if (!disposed) { + setSettings(nextSettings); + setRegistry(nextRegistry); + } + }) + .catch((loadError) => { + if (!disposed) { + setError( + loadError instanceof Error + ? loadError.message + : "Voice settings could not be loaded.", + ); + } + }); + return () => { + disposed = true; + }; + }, []); + + const saveEnabled = React.useCallback(async (enabled: boolean) => { + setBusy(true); + setError(null); + try { + const saved = await invokeTauri("set_tts_enabled", { + enabled, + }); + setSettings(saved); + } catch (saveError) { + try { + const state = await invokeTauri<{ tts_enabled: boolean }>( + "get_huddle_state", + ); + setSettings((current) => + current + ? { ...current, agentTextToSpeech: state.tts_enabled } + : current, + ); + } catch { + // Keep the last confirmed state when native reconciliation is + // unavailable; the visible save error makes the failure explicit. + } + setError( + saveError instanceof Error + ? saveError.message + : "Voice settings could not be saved.", + ); + } finally { + setBusy(false); + } + }, []); + + const savePocketVoice = React.useCallback(async (voiceKey: string) => { + setBusy(true); + setError(null); + try { + const saved = await invokeTauri("set_pocket_voice", { + voiceKey, + }); + setSettings(saved); + } catch (saveError) { + setError( + saveError instanceof Error + ? saveError.message + : "Voice settings could not be saved.", + ); + } finally { + setBusy(false); + } + }, []); + + const voices = voicesForBackend(registry, "pocket"); + const selectedVoice = selectedVoiceForBackend( + settings?.voicePreferences ?? [], + voices, + ); + const enabled = settings?.agentTextToSpeech ?? true; + const controlsDisabled = !settings || busy || !enabled; + + return ( +
+ + +
+ + +
+ +

+ Read new agent messages aloud in the order they arrive. +

+
+ { + if (settings) void saveEnabled(checked); + }} + /> + + + +
+ + +
+

Voice

+

+ Voice files stay private on this device. +

+
+ +
+ + + + + + { + if (settings) void savePocketVoice(voiceKey); + }} + value={selectedVoice?.key} + > + {voices.map((voice) => ( + + {voiceOptionLabel(voice, voices)} + + ))} + + + + +
+ + +
+ + + + {error && ( +

+ {error} +

+ )} +
+
+ ); +} diff --git a/desktop/src/features/settings/ui/playbackSpeedPersistence.test.mjs b/desktop/src/features/settings/ui/playbackSpeedPersistence.test.mjs new file mode 100644 index 0000000000..779a22c2ce --- /dev/null +++ b/desktop/src/features/settings/ui/playbackSpeedPersistence.test.mjs @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + applyLoadedPlaybackSpeed, + commitPlaybackSpeed, +} from "./playbackSpeedPersistence.ts"; + +function deferred() { + let resolve; + let reject; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, reject, resolve }; +} + +function harness() { + const calls = []; + const errors = []; + const saves = []; + const speeds = []; + const state = { + confirmedSpeed: { current: 1 }, + desiredSpeed: { current: 1 }, + flushPromise: { current: null }, + hasLocalIntent: { current: false }, + }; + const callbacks = { + persist: (speed) => { + calls.push(speed); + const save = deferred(); + saves.push(save); + return save.promise; + }, + setError: (error) => errors.push(error), + setSpeed: (speed) => speeds.push(speed), + }; + return { callbacks, calls, errors, saves, speeds, state }; +} + +async function settle(state) { + while (state.flushPromise.current) { + await state.flushPromise.current; + } +} + +test("rapid intents persist serially and finish at the latest speed", async () => { + const h = harness(); + commitPlaybackSpeed(h.state, h.callbacks, 1.25); + commitPlaybackSpeed(h.state, h.callbacks, 1.5); + assert.deepEqual(h.calls, [1.25]); + + h.saves[0].resolve(); + await Promise.resolve(); + assert.deepEqual(h.calls, [1.25, 1.5]); + h.saves[1].resolve(); + await settle(h.state); + + assert.equal(h.state.confirmedSpeed.current, 1.5); + assert.equal(h.state.desiredSpeed.current, 1.5); +}); + +test("a stale failure does not roll back a newer successful intent", async () => { + const h = harness(); + commitPlaybackSpeed(h.state, h.callbacks, 1.25); + commitPlaybackSpeed(h.state, h.callbacks, 1.5); + + h.saves[0].reject(new Error("first failed")); + await Promise.resolve(); + assert.deepEqual(h.calls, [1.25, 1.5]); + h.saves[1].resolve(); + await settle(h.state); + + assert.equal(h.state.confirmedSpeed.current, 1.5); + assert.equal(h.state.desiredSpeed.current, 1.5); + assert.deepEqual(h.speeds, [1.25, 1.5]); + assert.deepEqual(h.errors, [null, null]); +}); + +test("the latest failure rolls back to the last confirmed speed", async () => { + const h = harness(); + commitPlaybackSpeed(h.state, h.callbacks, 1.25); + h.saves[0].reject(new Error("save failed")); + await settle(h.state); + + assert.equal(h.state.confirmedSpeed.current, 1); + assert.equal(h.state.desiredSpeed.current, 1); + assert.deepEqual(h.speeds, [1.25, 1]); + assert.equal(h.errors.at(-1), "Error: save failed"); +}); + +test("a delayed initial load cannot overwrite a local intent", async () => { + const h = harness(); + commitPlaybackSpeed(h.state, h.callbacks, 1.5); + applyLoadedPlaybackSpeed(h.state, h.callbacks, 1); + + assert.equal(h.state.desiredSpeed.current, 1.5); + assert.deepEqual(h.speeds, [1.5]); + + h.saves[0].resolve(); + await settle(h.state); + assert.equal(h.state.confirmedSpeed.current, 1.5); + assert.equal(h.state.desiredSpeed.current, 1.5); +}); diff --git a/desktop/src/features/settings/ui/playbackSpeedPersistence.ts b/desktop/src/features/settings/ui/playbackSpeedPersistence.ts new file mode 100644 index 0000000000..88240888e3 --- /dev/null +++ b/desktop/src/features/settings/ui/playbackSpeedPersistence.ts @@ -0,0 +1,75 @@ +type MutableValue = { + current: T; +}; + +export type PlaybackSpeedPersistenceState = { + confirmedSpeed: MutableValue; + desiredSpeed: MutableValue; + flushPromise: MutableValue | null>; + hasLocalIntent: MutableValue; +}; + +type PlaybackSpeedPersistenceCallbacks = { + persist: (speed: number) => Promise; + setSpeed: (speed: number) => void; + setError: (error: string | null) => void; +}; + +export function commitPlaybackSpeed( + state: PlaybackSpeedPersistenceState, + callbacks: PlaybackSpeedPersistenceCallbacks, + nextSpeed: number, +) { + state.hasLocalIntent.current = true; + state.desiredSpeed.current = nextSpeed; + callbacks.setSpeed(nextSpeed); + callbacks.setError(null); + ensurePlaybackSpeedFlush(state, callbacks); +} + +export function applyLoadedPlaybackSpeed( + state: PlaybackSpeedPersistenceState, + callbacks: Pick, + savedSpeed: number, +) { + if (state.hasLocalIntent.current) return; + state.confirmedSpeed.current = savedSpeed; + state.desiredSpeed.current = savedSpeed; + callbacks.setSpeed(savedSpeed); +} + +function ensurePlaybackSpeedFlush( + state: PlaybackSpeedPersistenceState, + callbacks: PlaybackSpeedPersistenceCallbacks, +) { + if (state.flushPromise.current) return; + + const flush = flushPlaybackSpeed(state, callbacks).finally(() => { + if (state.flushPromise.current !== flush) return; + state.flushPromise.current = null; + if (state.desiredSpeed.current !== state.confirmedSpeed.current) { + ensurePlaybackSpeedFlush(state, callbacks); + } + }); + state.flushPromise.current = flush; +} + +async function flushPlaybackSpeed( + state: PlaybackSpeedPersistenceState, + callbacks: PlaybackSpeedPersistenceCallbacks, +) { + while (state.desiredSpeed.current !== state.confirmedSpeed.current) { + const target = state.desiredSpeed.current; + try { + await callbacks.persist(target); + state.confirmedSpeed.current = target; + } catch (cause) { + if (state.desiredSpeed.current === target) { + state.desiredSpeed.current = state.confirmedSpeed.current; + callbacks.setSpeed(state.confirmedSpeed.current); + callbacks.setError(String(cause)); + return; + } + } + } +} diff --git a/desktop/src/features/settings/ui/voiceSettingsLogic.test.mjs b/desktop/src/features/settings/ui/voiceSettingsLogic.test.mjs new file mode 100644 index 0000000000..d9e5ce9d03 --- /dev/null +++ b/desktop/src/features/settings/ui/voiceSettingsLogic.test.mjs @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + selectedVoiceForBackend, + voiceOptionLabel, + voicesForBackend, +} from "./voiceSettingsLogic.ts"; + +const voice = (key, displayName, fallbackKey = "pocket:mary") => ({ + key, + displayName, + backend: "pocket", + backendName: "Pocket TTS", + availability: "bundled", + fallbackKey, + referenceFile: `${key}.wav`, + provenance: { + source: "bundled", + contentHash: null, + license: null, + sourceUrl: null, + }, +}); + +test("Pocket-only V1 filters the shared registry by backend", () => { + const registry = [ + voice("pocket:mary", "Mary", null), + { ...voice("siri:aaron", "Aaron"), backend: "siri" }, + ]; + assert.deepEqual( + voicesForBackend(registry, "pocket").map((entry) => entry.key), + ["pocket:mary"], + ); +}); + +test("local selection uses the first compatible qualified preference", () => { + const voices = [ + voice("pocket:mary", "Mary", null), + voice("pocket:marius", "Marius"), + ]; + assert.equal( + selectedVoiceForBackend( + ["siri:aaron", "pocket:marius", "pocket:mary"], + voices, + )?.key, + "pocket:marius", + ); +}); + +test("duplicate display labels remain distinct by content-derived key", () => { + const voices = [ + voice("pocket:imported:aaa", "Jim"), + voice("pocket:imported:bbb", "Jim"), + ]; + assert.equal( + selectedVoiceForBackend(["pocket:imported:bbb"], voices)?.key, + "pocket:imported:bbb", + ); + assert.equal(voiceOptionLabel(voices[0], voices), "Jim · aaa"); + assert.equal(voiceOptionLabel(voices[1], voices), "Jim · bbb"); +}); diff --git a/desktop/src/features/settings/ui/voiceSettingsLogic.ts b/desktop/src/features/settings/ui/voiceSettingsLogic.ts new file mode 100644 index 0000000000..30352f2d0f --- /dev/null +++ b/desktop/src/features/settings/ui/voiceSettingsLogic.ts @@ -0,0 +1,57 @@ +export type VoiceAvailability = + | "bundled" + | "installed" + | "downloadable" + | "unavailable"; + +export type VoiceRegistryEntry = { + key: string; + displayName: string; + backend: string; + backendName: string; + availability: VoiceAvailability; + fallbackKey: string | null; + referenceFile: string | null; + provenance: { + source: string; + contentHash: string | null; + license: string | null; + sourceUrl: string | null; + }; +}; + +export function voicesForBackend( + registry: readonly VoiceRegistryEntry[], + backend: string, +): VoiceRegistryEntry[] { + return registry.filter( + (voice) => + voice.backend === backend && + (voice.availability === "bundled" || voice.availability === "installed"), + ); +} + +export function selectedVoiceForBackend( + preferences: readonly string[], + voices: readonly VoiceRegistryEntry[], +): VoiceRegistryEntry | undefined { + for (const key of preferences) { + const voice = voices.find((candidate) => candidate.key === key); + if (voice) return voice; + } + return voices.find((voice) => voice.fallbackKey === null) ?? voices[0]; +} + +export function voiceOptionLabel( + voice: VoiceRegistryEntry, + voices: readonly VoiceRegistryEntry[], +): string { + const duplicateLabel = voices.some( + (candidate) => + candidate.key !== voice.key && + candidate.displayName === voice.displayName, + ); + if (!duplicateLabel) return voice.displayName; + const identitySuffix = voice.key.split(":").at(-1)?.slice(-8) ?? voice.key; + return `${voice.displayName} · ${identitySuffix}`; +} diff --git a/desktop/src/shared/api/relayChannelFilters.test.mjs b/desktop/src/shared/api/relayChannelFilters.test.mjs index 3519c82146..7b89799049 100644 --- a/desktop/src/shared/api/relayChannelFilters.test.mjs +++ b/desktop/src/shared/api/relayChannelFilters.test.mjs @@ -6,6 +6,7 @@ import { buildChannelAuxFilter, buildChannelReactionAuxFilter, buildChannelStructuralAuxFilter, + buildHuddleTtsLiveFilter, } from "./relayChannelFilters.ts"; const CHANNEL = "36411e44-0e2d-4cfe-bd6e-567eb169db9f"; @@ -14,6 +15,14 @@ const IDS = [ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", ]; +test("huddle TTS filter is future-only kind:9 with no same-second replay", () => { + assert.deepEqual(buildHuddleTtsLiveFilter(CHANNEL), { + kinds: [9], + "#h": [CHANNEL], + limit: 0, + }); +}); + // Regression: reaction (kind:7) and reaction-removal (kind:5) events carry only // an `e` tag, no channel `h` tag. An `#h`-scoped aux query never matches them, // so removed historical reactions reappear. The aux filters must key on `#e` diff --git a/desktop/src/shared/api/relayChannelFilters.ts b/desktop/src/shared/api/relayChannelFilters.ts index d0c7e7938e..9acaef45c5 100644 --- a/desktop/src/shared/api/relayChannelFilters.ts +++ b/desktop/src/shared/api/relayChannelFilters.ts @@ -6,6 +6,7 @@ import { KIND_DELETION, KIND_NIP29_DELETE_EVENT, KIND_REACTION, + KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_EDIT, } from "@/shared/constants/kinds"; import type { RelaySubscriptionFilter } from "@/shared/api/relayClientShared"; @@ -40,6 +41,17 @@ export function buildChannelFilter( return filter; } +/** Strictly live huddle message filter: zero stored rows, future kind:9 only. */ +export function buildHuddleTtsLiveFilter( + channelId: string, +): RelaySubscriptionFilter { + return { + kinds: [KIND_STREAM_MESSAGE], + "#h": [channelId], + limit: 0, + }; +} + /** * History filter for cold-load and scrollback: message kinds *only*, so the * `limit` budget buys visible message depth. Auxiliary events (reactions, diff --git a/desktop/src/shared/api/relayClientSession.ts b/desktop/src/shared/api/relayClientSession.ts index 84ee10b68d..6be7fe04af 100644 --- a/desktop/src/shared/api/relayClientSession.ts +++ b/desktop/src/shared/api/relayClientSession.ts @@ -445,8 +445,9 @@ export class RelayClient { async subscribeLive( filter: RelaySubscriptionFilter, onEvent: (event: RelayEvent) => void, + options?: { replayMissedHistory?: boolean }, ) { - return this.subscribe(filter, onEvent); + return this.subscribe(filter, onEvent, options); } async subscribeToChannelMentionEvents( @@ -600,6 +601,7 @@ export class RelayClient { private async subscribe( filter: RelaySubscriptionFilter, onEvent: (event: RelayEvent) => void, + options?: { replayMissedHistory?: boolean }, ) { await this.ensureConnected(); @@ -621,6 +623,10 @@ export class RelayClient { mode: "live", filter, onEvent, + replayMissedHistory: options?.replayMissedHistory, + lastSeenCreatedAt: options?.replayMissedHistory + ? Math.floor(Date.now() / 1_000) + : undefined, resolveReady, }); diff --git a/desktop/src/shared/api/relayClientShared.ts b/desktop/src/shared/api/relayClientShared.ts index 9108f7b6d7..7559478b3d 100644 --- a/desktop/src/shared/api/relayClientShared.ts +++ b/desktop/src/shared/api/relayClientShared.ts @@ -58,6 +58,7 @@ type LiveSubscription = { mode: "live"; filter: RelaySubscriptionFilter; onEvent: (event: RelayEvent) => void; + replayMissedHistory?: boolean; resolveReady?: () => void; lastSeenCreatedAt?: number; closedRetryAttempt?: number; diff --git a/desktop/src/shared/api/relayReconnectReplay.test.mjs b/desktop/src/shared/api/relayReconnectReplay.test.mjs index 54254e89de..922d8ade49 100644 --- a/desktop/src/shared/api/relayReconnectReplay.test.mjs +++ b/desktop/src/shared/api/relayReconnectReplay.test.mjs @@ -5,6 +5,7 @@ import { buildReconnectReplayFilter, replayLiveSubscriptions, REPLAY_BATCH_SIZE, + shouldPageReconnectReplay, } from "./relayReconnectReplay.ts"; import { buildChannelFilter } from "./relayChannelFilters.ts"; @@ -113,6 +114,32 @@ test("reconnect replay caps large steady-state limits", () => { }); }); +test("reconnect replay preserves the live-only zero-history contract", () => { + const filter = { + kinds: [9], + "#h": ["channel-1"], + limit: 0, + }; + + assert.deepEqual(replayFilter(filter, 123), { + kinds: [9], + "#h": ["channel-1"], + limit: 0, + since: 123, + }); +}); + +test("missed-history replay is explicit for live-only subscriptions", () => { + const filter = { + kinds: [9], + "#h": ["channel-1"], + limit: 0, + }; + + assert.equal(shouldPageReconnectReplay(filter), false); + assert.equal(shouldPageReconnectReplay(filter, true), true); +}); + test("reconnect replay keeps the stricter existing since window", () => { const filter = { kinds: [9], diff --git a/desktop/src/shared/api/relayReconnectReplay.ts b/desktop/src/shared/api/relayReconnectReplay.ts index 74b752f666..6a09d1c549 100644 --- a/desktop/src/shared/api/relayReconnectReplay.ts +++ b/desktop/src/shared/api/relayReconnectReplay.ts @@ -69,7 +69,11 @@ export function buildReconnectReplayFilter( return replayFilter; } -export function shouldPageReconnectReplay(filter: RelaySubscriptionFilter) { +export function shouldPageReconnectReplay( + filter: RelaySubscriptionFilter, + replayMissedHistory = false, +) { + if (replayMissedHistory) return true; return ( filter.limit > 0 && Array.isArray(filter["#h"]) && @@ -176,7 +180,10 @@ export async function replayLiveSubscriptions({ ); const shouldPageReplay = replaySince !== undefined && - shouldPageReconnectReplay(subscription.filter); + shouldPageReconnectReplay( + subscription.filter, + subscription.replayMissedHistory, + ); return { subId, subscription, replaySince, shouldPageReplay }; }); diff --git a/desktop/src/shared/api/tauri.ts b/desktop/src/shared/api/tauri.ts index c57525480e..6eb43c6197 100644 --- a/desktop/src/shared/api/tauri.ts +++ b/desktop/src/shared/api/tauri.ts @@ -1197,6 +1197,14 @@ export const setPreventSleepActive = (active: boolean) => export const setAgentManagedProfiles = (enabled: boolean) => invokeTauri("set_agent_managed_profiles", { enabled }); +export function getTtsPlaybackSpeed(): Promise { + return invokeTauri("get_tts_playback_speed"); +} + +export function setTtsPlaybackSpeed(speed: number): Promise { + return invokeTauri("set_tts_playback_speed", { speed }); +} + /** Returns true on macOS, Windows, and Linux AppImage installs. * Returns false on Linux non-AppImage packages (e.g. .deb) where * Tauri's updater cannot swap the binary. */ diff --git a/desktop/src/testing/e2eBridge.ts b/desktop/src/testing/e2eBridge.ts index 03c05fe877..291763e87e 100644 --- a/desktop/src/testing/e2eBridge.ts +++ b/desktop/src/testing/e2eBridge.ts @@ -141,6 +141,11 @@ type MockSearchProfileSeed = { type E2eConfig = { mode?: "mock" | "relay"; mock?: { + ttsSettings?: { + version: number; + agentTextToSpeech: boolean; + voicePreferences: string[]; + }; /** Advertised HEAD for the first mock project without adding that branch. */ projectHeadBranch?: string; /** Builderlab account returned by hosted-community onboarding. Null/omitted = signed out. */ @@ -2793,6 +2798,7 @@ let mockClosedChannelLiveSubscription = false; const realSockets = new Map(); let mockManagedAgents: MockManagedAgent[] = []; let mockManagedAgentRuntimes: MockManagedAgentRuntimeRow[] = []; +let mockTtsPlaybackSpeed = 1; // Mutable `save_subscriptions` table mirror — TEST-ONLY. // @@ -9430,6 +9436,99 @@ export function maybeInstallE2eTauriMocks() { window.__BUZZ_E2E_COMMAND_LOG__?.push({ command, payload }); switch (command) { + case "get_tts_settings": + return ( + activeConfig?.mock?.ttsSettings ?? { + version: 1, + agentTextToSpeech: true, + voicePreferences: ["pocket:mary"], + } + ); + case "list_voice_registry": + return [ + { + key: "pocket:mary", + displayName: "Mary", + backend: "pocket", + backendName: "Pocket TTS", + availability: "bundled", + fallbackKey: null, + referenceFile: "reference_sample.wav", + provenance: { + source: "bundled", + contentHash: + "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f", + license: "CC-BY-4.0", + sourceUrl: "https://datashare.ed.ac.uk/handle/10283/3443", + }, + }, + { + key: "pocket:marius", + displayName: "Marius", + backend: "pocket", + backendName: "Pocket TTS", + availability: "bundled", + fallbackKey: "pocket:mary", + referenceFile: "marius.wav", + provenance: { + source: "bundled", + contentHash: + "076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16", + license: "CC0-1.0", + sourceUrl: + "https://huggingface.co/kyutai/tts-voices/blob/323332d33f997de8394f24a193e1a76df720e01a/voice-donations/Selfie.wav", + }, + }, + ]; + case "set_tts_enabled": { + const enabled = (payload as { enabled?: boolean })?.enabled; + if (typeof enabled !== "boolean") + throw new Error("Missing text-to-speech enabled state"); + const settings = { + version: 1, + agentTextToSpeech: enabled, + voicePreferences: activeConfig?.mock?.ttsSettings + ?.voicePreferences ?? ["pocket:mary"], + }; + if (activeConfig) { + activeConfig.mock ??= {}; + activeConfig.mock.ttsSettings = settings; + } + return settings; + } + case "set_pocket_voice": { + const voiceKey = (payload as { voiceKey?: string })?.voiceKey; + if (!voiceKey) throw new Error("Missing Pocket voice key"); + const current = activeConfig?.mock?.ttsSettings ?? { + version: 1, + agentTextToSpeech: true, + voicePreferences: ["pocket:mary"], + }; + const firstPocketIndex = current.voicePreferences.findIndex((key) => + key.startsWith("pocket:"), + ); + const preferences = current.voicePreferences.filter( + (key) => !key.startsWith("pocket:"), + ); + preferences.splice( + firstPocketIndex < 0 ? preferences.length : firstPocketIndex, + 0, + voiceKey, + ); + const settings = { ...current, voicePreferences: preferences }; + if (activeConfig) { + activeConfig.mock ??= {}; + activeConfig.mock.ttsSettings = settings; + } + return settings; + } + case "preview_pocket_voice": + return null; + case "get_tts_playback_speed": + return mockTtsPlaybackSpeed; + case "set_tts_playback_speed": + mockTtsPlaybackSpeed = (payload as { speed: number }).speed; + return null; case "get_builderlab_auth": return activeConfig?.mock?.builderlabAuth ?? null; case "start_builderlab_login": { diff --git a/desktop/tests/e2e/voice-settings.spec.ts b/desktop/tests/e2e/voice-settings.spec.ts new file mode 100644 index 0000000000..6904484776 --- /dev/null +++ b/desktop/tests/e2e/voice-settings.spec.ts @@ -0,0 +1,81 @@ +import { expect, test } from "@playwright/test"; + +import { waitForAnimations } from "../helpers/animations"; +import { installMockBridge } from "../helpers/bridge"; +import { openSettings } from "../helpers/settings"; + +const SCREENSHOT_PATH = "test-results/voice-settings/pocket-voices.png"; + +test.describe("Pocket voice settings", () => { + test.use({ viewport: { width: 1100, height: 760 } }); + + test("selects and retains a bundled voice while text to speech is off", async ({ + page, + }) => { + await installMockBridge(page); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await openSettings(page, "voice"); + + const card = page.getByTestId("settings-voice"); + await expect(card).toBeVisible(); + await expect( + page.getByText("Agent text to speech", { exact: true }), + ).toBeVisible(); + + await page.getByTestId("pocket-voice-selector").click(); + await page.getByRole("menuitemradio", { name: "Marius" }).click(); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Marius", + ); + + await page.getByTestId("agent-text-to-speech-toggle").click(); + await expect( + page.getByTestId("agent-text-to-speech-toggle"), + ).toHaveAttribute("aria-checked", "false"); + await expect(page.getByTestId("pocket-voice-controls")).toHaveAttribute( + "aria-disabled", + "true", + ); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Marius", + ); + + const savedCommands = await page.evaluate(() => + (window.__BUZZ_E2E_COMMAND_LOG__ ?? []) + .filter((entry) => + ["set_pocket_voice", "set_tts_enabled"].includes(entry.command), + ) + .map((entry) => ({ command: entry.command, payload: entry.payload })), + ); + expect(savedCommands).toEqual([ + { + command: "set_pocket_voice", + payload: { voiceKey: "pocket:marius" }, + }, + { + command: "set_tts_enabled", + payload: { enabled: false }, + }, + ]); + }); + + test("captures the complete two-voice settings surface", async ({ page }) => { + await installMockBridge(page, { + ttsSettings: { + version: 1, + agentTextToSpeech: true, + voicePreferences: ["pocket:marius"], + }, + }); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await openSettings(page, "voice"); + + const card = page.getByTestId("settings-voice"); + await expect(card).toBeVisible(); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Marius", + ); + await waitForAnimations(page); + await card.screenshot({ path: SCREENSHOT_PATH }); + }); +}); diff --git a/desktop/tests/helpers/bridge.ts b/desktop/tests/helpers/bridge.ts index d65a260f30..5f1ce5ea9d 100644 --- a/desktop/tests/helpers/bridge.ts +++ b/desktop/tests/helpers/bridge.ts @@ -128,6 +128,11 @@ export type MockAgentMemoryListing = { }; type MockBridgeOptions = { + ttsSettings?: { + version: number; + agentTextToSpeech: boolean; + voicePreferences: string[]; + }; /** Advertised HEAD for the first mock project without adding that branch. */ projectHeadBranch?: string; /** Relay NIP-11 identity used to sign authoritative repository state. */ diff --git a/desktop/tests/helpers/settings.ts b/desktop/tests/helpers/settings.ts index a63b52453e..c26c0e9195 100644 --- a/desktop/tests/helpers/settings.ts +++ b/desktop/tests/helpers/settings.ts @@ -3,6 +3,7 @@ import { expect, type Page } from "@playwright/test"; type SettingsSection = | "profile" | "notifications" + | "voice" | "agents" | "channel-templates" | "compute" diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 4dd1142801..ae26fc6ba8 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -84,6 +84,9 @@ run_unit_tests() { run_test_step "buzz-auth unit tests" \ cargo test -p buzz-auth --lib -- --nocapture + run_test_step "buzz-voice tests" \ + cargo test -p buzz-voice --lib -- --nocapture + # buzz-db migrator/lint unit tests (no infra): guard the embedded-migrator # invariant (exactly the consolidated 0001; cutover/backfill stays an operator # script, not startup state) and the tenant-scoping lints. The Postgres-backed