diff --git a/Cargo.lock b/Cargo.lock index 3b60dc4579..2e3182beda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,6 +58,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" +dependencies = [ + "alsa-sys", + "bitflags 2.13.0", + "cfg-if 1.0.4", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -396,6 +418,16 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomic-write-file" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84790c55b5704b0d35130bf16a4ce22a8e70eb0ea773522557524d9a4852663d" +dependencies = [ + "nix 0.30.1", + "rand 0.9.4", +] + [[package]] name = "attohttpc" version = "0.30.1" @@ -1262,6 +1294,20 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "atomic-write-file", + "cpal", + "hex", + "serde", + "serde_json", + "sha2 0.11.0", + "sherpa-onnx", + "tempfile", +] + [[package]] name = "buzz-workflow" version = "0.1.0" @@ -1329,6 +1375,26 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "castaway" version = "0.2.4" @@ -1710,6 +1776,20 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.13.0", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + [[package]] name = "countdown-bot" version = "0.1.0" @@ -1733,6 +1813,36 @@ dependencies = [ "futures-io", ] +[[package]] +name = "cpal" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8942da362c0f0d895d7cac616263f2f9424edc5687364dfd1d25ef7eba506d7" +dependencies = [ + "alsa", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.62.2", +] + [[package]] name = "cpubits" version = "0.1.1" @@ -2151,6 +2261,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.11.0" @@ -2174,7 +2290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -4418,6 +4534,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "mach2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea" +dependencies = [ + "libc", +] + [[package]] name = "matchers" version = "0.2.0" @@ -5225,12 +5350,35 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags 2.13.0", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "ndk-context" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "negentropy" version = "0.5.0" @@ -5371,6 +5519,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.13.0", + "cfg-if 1.0.4", + "cfg_aliases", + "libc", +] + [[package]] name = "nix" version = "0.31.3" @@ -5691,6 +5851,54 @@ dependencies = [ "objc2-encode", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.0", + "libc", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.0", + "objc2", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -8049,6 +8257,28 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "sherpa-onnx" +version = "1.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b142d3f255cb4e4b7808ea25869db6f5714e0a3550da355234483b4db552055" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc951af03dc0653c0622158ca8a585a6f2bc43b7b06048cf0e5b5020005c227" +dependencies = [ + "bzip2", + "tar", + "ureq", +] + [[package]] name = "shlex" version = "1.3.0" @@ -9539,6 +9769,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.8" diff --git a/Cargo.toml b/Cargo.toml index 3ac7ee4cce..3268cfaf8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "crates/buzz-pair-relay", "crates/buzz-relay-mesh", "crates/buzz-dev-mcp", + "crates/buzz-voice", "examples/countdown-bot", ] exclude = ["desktop/src-tauri"] diff --git a/Justfile b/Justfile index bcef8983bc..e5601607ba 100644 --- a/Justfile +++ b/Justfile @@ -276,6 +276,7 @@ test-unit: #!/usr/bin/env bash if command -v cargo-nextest &>/dev/null; then cargo nextest run -p buzz-core -p buzz-auth --lib + cargo nextest run -p buzz-voice --lib # buzz-db migrator/lint tests: pure SQL-parsing unit tests (no infra). # They guard the embedded-migrator invariant (exactly the consolidated # 0001; cutover/backfill stays an operator script, not startup state) diff --git a/crates/buzz-voice/Cargo.toml b/crates/buzz-voice/Cargo.toml new file mode 100644 index 0000000000..4fbef7dccc --- /dev/null +++ b/crates/buzz-voice/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "buzz-voice" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Reusable local voice primitives for Buzz" + +[dependencies] +atomic-write-file = "0.3" +hex = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +sherpa-onnx = "1.12" + +[dev-dependencies] +tempfile = "3" + +[target.'cfg(target_os = "macos")'.dev-dependencies] +cpal = "0.17" diff --git a/crates/buzz-voice/src/imported.rs b/crates/buzz-voice/src/imported.rs new file mode 100644 index 0000000000..4c494e226d --- /dev/null +++ b/crates/buzz-voice/src/imported.rs @@ -0,0 +1,549 @@ +//! Device-local Pocket reference voice validation, canonicalization, and storage. + +use std::{ + fs, + io::Write, + path::{Path, PathBuf}, +}; + +use atomic_write_file::AtomicWriteFile; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +const MAX_SOURCE_BYTES: u64 = 25 * 1024 * 1024; +const MIN_SAMPLE_RATE: u32 = 8_000; +const MAX_SAMPLE_RATE: u32 = 96_000; +const MIN_DURATION_SECONDS: f64 = 2.0; +const MAX_DURATION_SECONDS: f64 = 30.0; +pub const CANONICAL_SAMPLE_RATE: u32 = 32_000; +const REGISTRY_VERSION: u32 = 1; +const REGISTRY_FILE: &str = "registry.json"; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ImportedVoice { + pub key: String, + pub display_name: String, + pub content_hash: String, + pub file_name: String, +} + +#[derive(Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct ImportedVoiceRegistry { + version: u32, + voices: Vec, +} + +#[derive(Clone, Debug)] +pub struct PocketVoiceLibrary { + root: PathBuf, +} + +impl PocketVoiceLibrary { + pub fn new(root: impl Into) -> Self { + Self { root: root.into() } + } + + pub fn root(&self) -> &Path { + &self.root + } + + fn registry_path(&self) -> PathBuf { + self.root.join(REGISTRY_FILE) + } + + pub fn load(&self) -> Result, String> { + let path = self.registry_path(); + if !path.exists() { + return Ok(Vec::new()); + } + let bytes = + fs::read(&path).map_err(|error| format!("could not read imported voices: {error}"))?; + let registry: ImportedVoiceRegistry = serde_json::from_slice(&bytes) + .map_err(|error| format!("imported voice registry is invalid: {error}"))?; + if registry.version > REGISTRY_VERSION { + return Err(format!( + "imported voice registry version {} is newer than this Buzz build supports", + registry.version + )); + } + Ok(registry + .voices + .into_iter() + .filter(valid_identity) + .filter(|voice| self.resolve_file(voice).is_ok()) + .collect()) + } + + fn save(&self, voices: &[ImportedVoice]) -> Result<(), String> { + ensure_storage_dir(&self.root)?; + let payload = serde_json::to_vec_pretty(&ImportedVoiceRegistry { + version: REGISTRY_VERSION, + voices: voices.to_vec(), + }) + .map_err(|error| format!("could not encode imported voice registry: {error}"))?; + atomic_write_restricted(&self.registry_path(), &payload) + .map_err(|error| format!("could not save imported voice registry: {error}")) + } + + pub fn resolve_file(&self, voice: &ImportedVoice) -> Result { + if !valid_identity(voice) { + return Err("Imported voice registry contains an invalid file identity".to_string()); + } + let path = self.root.join(&voice.file_name); + if !is_regular_file_without_symlink(&path) { + return Err(format!("Imported voice {} is missing", voice.display_name)); + } + let bytes = + fs::read(&path).map_err(|error| format!("could not verify imported voice: {error}"))?; + if hex::encode(Sha256::digest(bytes)) != voice.content_hash { + return Err(format!( + "Imported voice {} does not match its content identity", + voice.display_name + )); + } + Ok(path) + } + + pub fn find(&self, key: &str) -> Result, String> { + Ok(self.load()?.into_iter().find(|voice| voice.key == key)) + } + + pub fn import_path(&self, source: &Path) -> Result { + let metadata = fs::metadata(source) + .map_err(|error| format!("could not inspect selected WAV: {error}"))?; + if metadata.len() > MAX_SOURCE_BYTES { + return Err("Voice WAV must be 25 MB or smaller".to_string()); + } + let source_bytes = + fs::read(source).map_err(|error| format!("could not read selected WAV: {error}"))?; + let samples = decode_wav(&source_bytes)?; + let canonical_samples = resample_linear(&samples.samples, samples.sample_rate); + let canonical = encode_pcm16_wav(&canonical_samples, CANONICAL_SAMPLE_RATE); + let hash = hex::encode(Sha256::digest(&canonical)); + let key = format!("pocket:imported:{hash}"); + let file_name = format!("{hash}.wav"); + let display_name = source + .file_stem() + .and_then(|name| name.to_str()) + .map(str::trim) + .filter(|name| !name.is_empty()) + .unwrap_or("Imported voice") + .chars() + .take(80) + .collect::(); + + ensure_storage_dir(&self.root)?; + let file_path = self.root.join(&file_name); + let file_created = !file_path.exists(); + if file_created { + atomic_write_restricted(&file_path, &canonical) + .map_err(|error| format!("could not save imported voice audio: {error}"))?; + } else { + if !is_regular_file_without_symlink(&file_path) { + return Err("Imported voice storage contains an unsafe file entry".to_string()); + } + let existing = fs::read(&file_path) + .map_err(|error| format!("could not verify imported voice audio: {error}"))?; + if hex::encode(Sha256::digest(&existing)) != hash { + return Err("Imported voice storage contains mismatched audio data".to_string()); + } + } + + let mut imported = ImportedVoice { + key, + display_name, + content_hash: hash, + file_name, + }; + let mut voices = self.load()?; + if let Some(existing) = voices + .iter() + .find(|voice| voice.content_hash == imported.content_hash) + { + imported = existing.clone(); + } else { + voices.push(imported.clone()); + } + if let Err(error) = self.save(&voices) { + if file_created { + let _ = fs::remove_file(&file_path); + } + return Err(error); + } + Ok(imported) + } + + pub fn delete(&self, key: &str) -> Result<(), String> { + let mut voices = self.load()?; + let index = voices + .iter() + .position(|voice| voice.key == key) + .ok_or_else(|| format!("Unknown imported voice: {key}"))?; + let previous_voices = voices.clone(); + let removed = voices.remove(index); + self.save(&voices)?; + let path = self.root.join(removed.file_name); + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => { + self.save(&previous_voices).map_err(|rollback_error| { + format!( + "Imported voice audio could not be deleted ({error}), and its registry \ + entry could not be restored ({rollback_error})" + ) + })?; + Err(format!( + "Imported voice audio could not be deleted: {error}" + )) + } + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PcmStats { + pub sample_count: usize, + pub sample_rate: u32, + pub duration_seconds: f64, + pub peak: f32, + pub rms: f32, + pub non_silent_samples: usize, +} + +impl PcmStats { + pub fn analyze(samples: &[f32], sample_rate: u32) -> Self { + let peak = samples + .iter() + .filter(|sample| sample.is_finite()) + .fold(0.0_f32, |peak, sample| peak.max(sample.abs())); + let square_sum = samples + .iter() + .filter(|sample| sample.is_finite()) + .map(|sample| sample * sample) + .sum::(); + let rms = if samples.is_empty() { + 0.0 + } else { + (square_sum / samples.len() as f32).sqrt() + }; + Self { + sample_count: samples.len(), + sample_rate, + duration_seconds: if sample_rate == 0 { + 0.0 + } else { + samples.len() as f64 / f64::from(sample_rate) + }, + peak, + rms, + non_silent_samples: samples + .iter() + .filter(|sample| sample.is_finite() && sample.abs() >= 0.001) + .count(), + } + } + + pub fn is_non_silent(self) -> bool { + self.peak >= 0.001 && self.rms >= 0.0001 && self.non_silent_samples > 0 + } +} + +pub fn write_pcm16_wav(path: &Path, samples: &[f32], sample_rate: u32) -> Result<(), String> { + let bytes = encode_pcm16_wav(samples, sample_rate); + fs::write(path, bytes).map_err(|error| format!("could not write PCM evidence: {error}")) +} + +fn ensure_storage_dir(path: &Path) -> Result<(), String> { + fs::create_dir_all(path) + .map_err(|error| format!("could not create local voice storage: {error}"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o700)) + .map_err(|error| format!("could not restrict local voice storage: {error}"))?; + } + Ok(()) +} + +fn atomic_write_restricted(path: &Path, payload: &[u8]) -> Result<(), String> { + let resolved = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let mut file = AtomicWriteFile::open(&resolved) + .map_err(|error| format!("open {} for atomic write: {error}", resolved.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(fs::Permissions::from_mode(0o600)) + .map_err(|error| format!("set {} permissions: {error}", resolved.display()))?; + } + file.write_all(payload) + .map_err(|error| format!("write {}: {error}", resolved.display()))?; + file.commit() + .map_err(|error| format!("commit {}: {error}", resolved.display())) +} + +fn valid_hash(hash: &str) -> bool { + hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn valid_identity(voice: &ImportedVoice) -> bool { + valid_hash(&voice.content_hash) + && voice.key == format!("pocket:imported:{}", voice.content_hash) + && voice.file_name == format!("{}.wav", voice.content_hash) +} + +fn is_regular_file_without_symlink(path: &Path) -> bool { + fs::symlink_metadata(path) + .is_ok_and(|metadata| metadata.file_type().is_file() && !metadata.file_type().is_symlink()) +} + +#[derive(Debug)] +struct DecodedWav { + sample_rate: u32, + samples: Vec, +} + +fn decode_wav(bytes: &[u8]) -> Result { + if bytes.len() < 12 || &bytes[..4] != b"RIFF" || &bytes[8..12] != b"WAVE" { + return Err("Selected file is not a valid RIFF/WAVE file".to_string()); + } + let mut offset = 12usize; + let mut format = None; + let mut data = None; + while offset.checked_add(8).is_some_and(|end| end <= bytes.len()) { + let id = &bytes[offset..offset + 4]; + let size = + u32::from_le_bytes(bytes[offset + 4..offset + 8].try_into().unwrap_or([0; 4])) as usize; + let start = offset + 8; + let end = start.checked_add(size).ok_or("WAV chunk size overflow")?; + if end > bytes.len() { + return Err("Selected WAV contains a truncated chunk".to_string()); + } + if id == b"fmt " { + format = Some(&bytes[start..end]); + } else if id == b"data" { + data = Some(&bytes[start..end]); + } + offset = end + (size & 1); + } + let format = format.ok_or("Selected WAV has no format chunk")?; + let data = data.ok_or("Selected WAV has no audio data")?; + if format.len() < 16 { + return Err("Selected WAV has an invalid format chunk".to_string()); + } + let encoding = u16::from_le_bytes(format[0..2].try_into().unwrap_or([0; 2])); + let encoding = if encoding == 0xfffe && format.len() >= 40 { + u16::from_le_bytes(format[24..26].try_into().unwrap_or([0; 2])) + } else { + encoding + }; + let channels = u16::from_le_bytes(format[2..4].try_into().unwrap_or([0; 2])); + let sample_rate = u32::from_le_bytes(format[4..8].try_into().unwrap_or([0; 4])); + let block_align = u16::from_le_bytes(format[12..14].try_into().unwrap_or([0; 2])) as usize; + let bits = u16::from_le_bytes(format[14..16].try_into().unwrap_or([0; 2])); + if channels != 1 { + return Err("Voice WAV must be mono".to_string()); + } + if !(MIN_SAMPLE_RATE..=MAX_SAMPLE_RATE).contains(&sample_rate) { + return Err("Voice WAV sample rate must be between 8 and 96 kHz".to_string()); + } + let bytes_per_sample = usize::from(bits.div_ceil(8)); + if block_align != bytes_per_sample || block_align == 0 || data.len() % block_align != 0 { + return Err("Voice WAV has invalid sample alignment".to_string()); + } + if !matches!((encoding, bits), (1, 8 | 16 | 24 | 32) | (3, 32)) { + return Err("Voice WAV must contain PCM or 32-bit float audio".to_string()); + } + let frames = data.len() / block_align; + let duration = frames as f64 / f64::from(sample_rate); + if !(MIN_DURATION_SECONDS..=MAX_DURATION_SECONDS).contains(&duration) { + return Err("Voice WAV must be between 2 and 30 seconds long".to_string()); + } + + let mut samples = Vec::with_capacity(frames); + for chunk in data.chunks_exact(block_align) { + let sample = match (encoding, bits) { + (1, 8) => (f32::from(chunk[0]) - 128.0) / 128.0, + (1, 16) => f32::from(i16::from_le_bytes([chunk[0], chunk[1]])) / 32768.0, + (1, 24) => { + let raw = i32::from_le_bytes([ + chunk[0], + chunk[1], + chunk[2], + if chunk[2] & 0x80 == 0 { 0 } else { 0xff }, + ]); + raw as f32 / 8_388_608.0 + } + (1, 32) => { + i32::from_le_bytes(chunk.try_into().map_err(|_| "invalid PCM sample")?) as f32 + / 2_147_483_648.0 + } + (3, 32) => f32::from_le_bytes( + chunk + .try_into() + .map_err(|_| "invalid floating-point sample")?, + ), + _ => unreachable!(), + }; + if !sample.is_finite() { + return Err("Voice WAV contains non-finite samples".to_string()); + } + samples.push(sample.clamp(-1.0, 1.0)); + } + let stats = PcmStats::analyze(&samples, sample_rate); + if !stats.is_non_silent() { + return Err("Voice WAV is silent or too quiet to clone".to_string()); + } + Ok(DecodedWav { + sample_rate, + samples, + }) +} + +fn resample_linear(samples: &[f32], source_rate: u32) -> Vec { + if source_rate == CANONICAL_SAMPLE_RATE { + return samples.to_vec(); + } + let output_len = ((samples.len() as u64 * u64::from(CANONICAL_SAMPLE_RATE) + + u64::from(source_rate) / 2) + / u64::from(source_rate)) as usize; + (0..output_len) + .map(|index| { + let source = index as f64 * f64::from(source_rate) / f64::from(CANONICAL_SAMPLE_RATE); + let left = source.floor() as usize; + let fraction = (source - left as f64) as f32; + let a = samples[left.min(samples.len() - 1)]; + let b = samples[(left + 1).min(samples.len() - 1)]; + a + (b - a) * fraction + }) + .collect() +} + +fn encode_pcm16_wav(samples: &[f32], sample_rate: u32) -> Vec { + let data_len = (samples.len() * 2) as u32; + let mut bytes = Vec::with_capacity(44 + data_len as usize); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + data_len).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16_u32.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&sample_rate.to_le_bytes()); + bytes.extend_from_slice(&(sample_rate * 2).to_le_bytes()); + bytes.extend_from_slice(&2_u16.to_le_bytes()); + bytes.extend_from_slice(&16_u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&data_len.to_le_bytes()); + for sample in samples { + let value = (sample.clamp(-1.0, 1.0) * f32::from(i16::MAX)).round() as i16; + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture(sample_rate: u32, seconds: usize, amplitude: f32) -> Vec { + let samples = (0..sample_rate as usize * seconds) + .map(|index| { + amplitude + * (std::f32::consts::TAU * 220.0 * index as f32 / sample_rate as f32).sin() + }) + .collect::>(); + encode_pcm16_wav(&samples, sample_rate) + } + + #[test] + fn imports_persists_reloads_and_deletes_canonical_voice() { + let temp = tempfile::tempdir().expect("temp voice workspace"); + let source = temp.path().join("My voice.wav"); + fs::write(&source, fixture(44_100, 2, 0.5)).expect("write source"); + let library = PocketVoiceLibrary::new(temp.path().join("library")); + + let imported = library.import_path(&source).expect("import voice"); + assert!(imported.key.starts_with("pocket:imported:")); + assert_eq!(imported.display_name, "My voice"); + + let relaunched = PocketVoiceLibrary::new(library.root()); + assert_eq!( + relaunched.load().expect("reload registry"), + vec![imported.clone()] + ); + let stored = relaunched + .resolve_file(&imported) + .expect("resolve stored voice"); + let decoded = decode_wav(&fs::read(&stored).expect("read stored voice")) + .expect("decode canonical voice"); + assert_eq!(decoded.sample_rate, CANONICAL_SAMPLE_RATE); + assert_eq!(decoded.samples.len(), CANONICAL_SAMPLE_RATE as usize * 2); + + assert_eq!( + relaunched.import_path(&source).expect("idempotent import"), + imported + ); + assert_eq!(relaunched.load().expect("deduplicated registry").len(), 1); + + relaunched.delete(&imported.key).expect("delete voice"); + assert!(relaunched.load().expect("empty registry").is_empty()); + assert!(!stored.exists()); + } + + #[test] + fn invalid_unsupported_and_silent_files_do_not_mutate_registry() { + let temp = tempfile::tempdir().expect("temp voice workspace"); + let library = PocketVoiceLibrary::new(temp.path().join("library")); + + let garbage = temp.path().join("garbage.wav"); + fs::write(&garbage, b"not a wave").expect("write garbage"); + assert!(library + .import_path(&garbage) + .expect_err("garbage rejected") + .contains("RIFF/WAVE")); + + let silent = temp.path().join("silent.wav"); + fs::write(&silent, fixture(32_000, 2, 0.0)).expect("write silence"); + assert!(library + .import_path(&silent) + .expect_err("silence rejected") + .contains("silent")); + + let mut stereo = fixture(32_000, 2, 0.5); + stereo[22..24].copy_from_slice(&2_u16.to_le_bytes()); + let stereo_path = temp.path().join("stereo.wav"); + fs::write(&stereo_path, stereo).expect("write stereo"); + assert!(library + .import_path(&stereo_path) + .expect_err("stereo rejected") + .contains("mono")); + + let mut unsupported = fixture(32_000, 2, 0.5); + unsupported[20..22].copy_from_slice(&6_u16.to_le_bytes()); + let unsupported_path = temp.path().join("unsupported.wav"); + fs::write(&unsupported_path, unsupported).expect("write unsupported"); + assert!(library + .import_path(&unsupported_path) + .expect_err("unsupported rejected") + .contains("PCM or 32-bit float")); + + assert!(library.load().expect("unchanged registry").is_empty()); + } + + #[test] + fn pcm_analysis_distinguishes_signal_from_silence() { + let signal = (0..24_000) + .map(|index| (std::f32::consts::TAU * 440.0 * index as f32 / 24_000.0).sin() * 0.5) + .collect::>(); + let signal_stats = PcmStats::analyze(&signal, 24_000); + assert!(signal_stats.is_non_silent()); + assert_eq!(signal_stats.duration_seconds, 1.0); + assert!(signal_stats.peak > 0.49); + assert!(signal_stats.rms > 0.3); + + let silence = vec![0.0; 24_000]; + assert!(!PcmStats::analyze(&silence, 24_000).is_non_silent()); + } +} diff --git a/crates/buzz-voice/src/lib.rs b/crates/buzz-voice/src/lib.rs new file mode 100644 index 0000000000..258c4590cc --- /dev/null +++ b/crates/buzz-voice/src/lib.rs @@ -0,0 +1,4 @@ +//! Reusable local voice primitives for Buzz. + +pub mod imported; +pub mod pocket; diff --git a/crates/buzz-voice/src/pocket.rs b/crates/buzz-voice/src/pocket.rs new file mode 100644 index 0000000000..ae235b93eb --- /dev/null +++ b/crates/buzz-voice/src/pocket.rs @@ -0,0 +1,640 @@ +//! Pocket TTS engine wrapper around sherpa-onnx's `OfflineTts`. +//! +//! Pocket TTS is a small (~473 MB fp32 ONNX) zero-shot voice-cloning TTS +//! model from Kyutai that runs quickly on CPU via sherpa-onnx. +//! +//! Buzz uses full-precision fp32 sessions because a direct same-runtime A/B +//! (k2-fsa/sherpa-onnx#3172) found the ~189 MB int8 ONNX export audibly +//! degraded output quality. +//! +//! ## Attribution +//! +//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. +//! arXiv:2509.06926. Original repository: . +//! Licensed CC-BY-4.0. +//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. +//! - **ONNX export**: KevinAHM — +//! . CC-BY-4.0. +//! - **sherpa-onnx repackage**: csukuangfj / k2-fsa — +//! . +//! Repackages KevinAHM's export with the file layout sherpa-onnx's +//! `OfflineTtsPocketModelConfig` expects. CC-BY-4.0. +//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary +//! (f, conversation)" preset from the Kyutai TTS demo +//! (), which maps to `vctk/p333_023_enhanced.wav` +//! in . CC-BY-4.0, base recording +//! from the VCTK corpus, enhanced by ai-coustics. +//! +//! Buzz ships these files unmodified; see the on-disk `MODEL_LICENSE.txt` +//! sidecar written by `huddle::models` during install for the canonical +//! CC-BY-4.0 §3(a)(1) attribution block. +//! +//! ## Engine-module contract (see `huddle::tts`) +//! +//! `pocket.rs` exposes a fixed surface used by `tts.rs`. Mirroring this +//! contract is what lets the TTS pipeline stay engine-agnostic: +//! +//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. +//! - `DEFAULT_VOICE: &str` — default voice name (without extension). +//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. +//! - `load_text_to_speech(model_dir)` → `Result` +//! - `load_voice_style(path)` → `Result` +//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` +//! → `Result, String>` +//! +//! `lang` and `steps` are accepted for API compatibility with the previous +//! Kokoro engine but are unused — Pocket TTS does its own language ID from +//! the input text and is not a diffusion model (consistency LM, one step). +//! There is no speed knob: sherpa-onnx's `GenerationConfig.speed` is only +//! read by some model families (vits), never by the Pocket impl +//! (`offline-tts-pocket-impl.h` — zero references), and upstream pocket-tts +//! has no speed parameter either. + +use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; + +use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; + +// ── Engine-module contract: public consts ───────────────────────────────────── + +/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, +/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. +pub const SAMPLE_RATE: u32 = 24_000; + +/// Name (without extension) of the bundled reference voice. The model directory +/// is expected to contain `.` after install. +pub const DEFAULT_VOICE: &str = "reference_sample"; + +/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the +/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, +/// not a precomputed embedding, so the extension change is honest. +pub const VOICE_FILE_EXT: &str = "wav"; + +// ── Tuning ──────────────────────────────────────────────────────────────────── + +/// Single-threaded ONNX execution for predictable CPU contention with the STT +/// pipeline. Matches `STT_NUM_THREADS` in `stt.rs`; raise only if a benchmark +/// argues for it. +const TTS_NUM_THREADS: i32 = 1; + +/// LRU cache size for cloned voice embeddings inside the sherpa-onnx engine. +/// We bind to one voice per pipeline today, but the upstream example uses 16 +/// and the cost is negligible — keep room for future multi-voice support. +const VOICE_EMBEDDING_CACHE_CAPACITY: i32 = 16; + +/// Pocket TTS is a consistency-based LM. Generation quality saturates at one +/// denoising step — the upstream `GenerationConfig` default of 5 multiplies +/// synthesis time by ~5× with no audible benefit on this model. +const SYNTH_NUM_STEPS: i32 = 1; + +/// Leave the generated audio's silences untouched (1.0 is the identity). +/// +/// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding +/// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and +/// multiplies its length by this factor. The reference Pocket TTS pipeline +/// preserves natural clause breaks, breaths, and punctuation pauses, so the +/// identity scale keeps those interior silences intact. +const SYNTH_SILENCE_SCALE: f32 = 1.0; + +/// sherpa-onnx upstream default for `max_frames` (LM steps), in +/// `offline-tts-pocket-impl.h:Generate`. 500 steps ≈ 40 s of audio at the +/// Mimi 12.5 Hz frame rate. Referenced only by the regression test below; +/// production code path never raises (or even reads) this value — we just +/// leave sherpa-onnx's own default in place by not setting the override. +#[cfg(test)] +const SHERPA_ONNX_MAX_FRAMES_DEFAULT: i32 = 500; + +/// Tight `max_frames` we ask for on short prompts to bound the +/// original "monster breathing" runaway. 100 LM steps ≈ 8 s of audio — +/// roomy for any one-to-four-word utterance the user is likely to elicit +/// while still well short of the 40 s upstream default. Chosen with slack so +/// we never *truncate* a legitimate short reply. +const SHORT_PROMPT_MAX_FRAMES: i32 = 100; + +/// Word-count threshold (inclusive) below which we cap `max_frames` tighter +/// than the upstream default. Above this threshold we leave sherpa-onnx's +/// generation limits in place because dropping `frames_after_eos` below the +/// upstream default of 3 can clip the leading audio of multi-clause sentences. +const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; + +/// sherpa-onnx's documented `frames_after_eos` default. We deliberately do +/// *not* override this knob because values below the upstream default of 3 can +/// clip the leading audio of multi-clause sentences. The constant exists only +/// for the invariant test below. Source: `offline-tts-pocket-impl.h:Generate`. +#[cfg(test)] +const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; + +// ── ONNX file names (five Pocket TTS sessions plus two JSON tables) ─────────── + +const FILE_LM_MAIN: &str = "lm_main.onnx"; +const FILE_LM_FLOW: &str = "lm_flow.onnx"; +const FILE_ENCODER: &str = "encoder.onnx"; +const FILE_DECODER: &str = "decoder.onnx"; +const FILE_TEXT_COND: &str = "text_conditioner.onnx"; +const FILE_VOCAB: &str = "vocab.json"; +const FILE_TOKEN_SCORES: &str = "token_scores.json"; + +// ── Voice style ─────────────────────────────────────────────────────────────── + +/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. +/// +/// Pocket TTS takes a reference waveform per generation call (not a +/// precomputed style embedding), so we keep the samples in memory and clone +/// the small `Vec` into each `GenerationConfig` rather than re-reading the +/// WAV from disk on every sentence. +#[derive(Debug, Clone)] +pub struct VoiceStyle { + samples: Vec, + sample_rate: i32, +} + +/// Load a reference voice WAV from disk. +/// +/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS +/// resamples internally using `reference_sample_rate`. The bundled +/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. +pub fn load_voice_style(path: &Path) -> Result { + let path_str = path + .to_str() + .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; + let wave = Wave::read(path_str) + .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; + let samples = wave.samples().to_vec(); + if samples.is_empty() { + return Err(format!("voice WAV is empty: {}", path.display())); + } + Ok(VoiceStyle { + samples, + sample_rate: wave.sample_rate(), + }) +} + +// ── Engine ──────────────────────────────────────────────────────────────────── + +/// Pocket TTS engine handle. Cheap to construct (one `OfflineTts::create` +/// call). Owned by the TTS worker thread for the lifetime of a huddle session. +/// +/// `OfflineTts` does not implement `Debug`, so we don't derive it here — the +/// pipeline only needs to move the engine into the worker thread and call +/// `synth_chunk` on it, never to print it. +pub struct PocketTts { + inner: OfflineTts, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum SynthesisOutcome { + Complete(Vec), + Interrupted, +} + +/// Build the Pocket TTS engine from the model directory installed by +/// `huddle::models`. Returns `Err` if any expected ONNX or JSON file is +/// missing — readiness is normally enforced by `is_tts_ready` upstream, but +/// the check is repeated here so a manually-modified model dir produces a +/// clear error string instead of an opaque sherpa-onnx `None`. +pub fn load_text_to_speech(model_dir: &str) -> Result { + let dir = PathBuf::from(model_dir); + for name in [ + FILE_LM_MAIN, + FILE_LM_FLOW, + FILE_ENCODER, + FILE_DECODER, + FILE_TEXT_COND, + FILE_VOCAB, + FILE_TOKEN_SCORES, + ] { + let p = dir.join(name); + if !p.is_file() { + return Err(format!("missing Pocket TTS file: {}", p.display())); + } + } + + let to_str = |name: &str| -> String { dir.join(name).to_string_lossy().into_owned() }; + + // Build the config by mutating defaults — mirrors `stt.rs` and stays + // resilient if sherpa-onnx adds unrelated model-family fields. + let mut cfg = OfflineTtsConfig::default(); + cfg.model.pocket.lm_main = Some(to_str(FILE_LM_MAIN)); + cfg.model.pocket.lm_flow = Some(to_str(FILE_LM_FLOW)); + cfg.model.pocket.encoder = Some(to_str(FILE_ENCODER)); + cfg.model.pocket.decoder = Some(to_str(FILE_DECODER)); + cfg.model.pocket.text_conditioner = Some(to_str(FILE_TEXT_COND)); + cfg.model.pocket.vocab_json = Some(to_str(FILE_VOCAB)); + cfg.model.pocket.token_scores_json = Some(to_str(FILE_TOKEN_SCORES)); + cfg.model.pocket.voice_embedding_cache_capacity = VOICE_EMBEDDING_CACHE_CAPACITY; + cfg.model.num_threads = TTS_NUM_THREADS; + // Explicit — defaults are not part of the API contract, and noisy debug + // logging in release builds would be expensive on every synthesized chunk. + cfg.model.debug = false; + + let inner = OfflineTts::create(&cfg) + .ok_or_else(|| "OfflineTts::create returned None for Pocket TTS".to_string())?; + Ok(PocketTts { inner }) +} + +// ── Prompt preparation ──────────────────────────────────────────────────────── + +/// Result of [`prepare_pocket_prompt`]: a synthesizer-ready prompt plus the +/// per-call generation overrides derived from the original text. +/// +/// `None` for either override means "leave sherpa-onnx's documented default +/// in place". The pipeline only sets `max_frames` for short prompts, bounding +/// runaway generation without disturbing the rest of the LM sampling envelope. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct PreparedPrompt { + /// Text to hand to `OfflineTts::generate_with_config`. Capitalized, + /// punctuation-terminated, and prefixed with the production `. ` cold-start + /// mitigation. + pub text: String, + /// Value to pass via `GenerationConfig.extra["max_frames"]`, or `None` to + /// keep the upstream default of 500 LM steps. We only override on short + /// prompts where we have a tight expectation on output length. + pub max_frames: Option, +} + +/// Mirror of the *text-preparation* half of upstream +/// `pocket_tts.models.tts_model.prepare_text_prompt`. Sherpa-onnx's C++ +/// Pocket TTS impl does not run these preparation steps, so short / +/// unpunctuated / lowercase inputs can trigger up to 40 s of runaway +/// generation when the EOS logit never crosses its threshold. +/// +/// We mirror the upstream Python recipe by: +/// +/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but +/// cheap to re-check after sentence splitting). +/// 2. Capitalize the first letter. +/// 3. Append `.` if the text doesn't end in punctuation. +/// 4. If fewer than five words, return a tight +/// [`SHORT_PROMPT_MAX_FRAMES`] cap so the LM can't run away if EOS still +/// doesn't fire. +/// +/// Buzz also prepends `". "` to every prompt. This sacrificial punctuation +/// lets the autoregressive model settle before it renders the first real word. +/// +/// We do **not** override `frames_after_eos` — sherpa-onnx's default of 3 +/// is what we want. An earlier version set it to 1 on long inputs, which +/// clipped the leading audio of multi-clause sentences ("first 'yep' is +/// just static" regression). Tests `prepare_prompt_never_lowers_frames_…` +/// lock this in. +/// +/// Returns `None` only if the input is empty after trimming — caller should +/// skip synthesis in that case. +pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { + let trimmed = input.trim(); + if trimmed.is_empty() { + return None; + } + + // Collapse stray double-spaces / embedded newlines that may slip past + // `preprocess_for_tts` when sentences are spliced back together. + let mut cleaned = String::with_capacity(trimmed.len()); + let mut last_was_space = false; + for ch in trimmed.chars() { + let is_ws = ch.is_whitespace(); + if is_ws { + if !last_was_space { + cleaned.push(' '); + } + last_was_space = true; + } else { + cleaned.push(ch); + last_was_space = false; + } + } + + // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). + let first = cleaned.chars().next().expect("cleaned non-empty above"); + if first.is_lowercase() { + let upper: String = first.to_uppercase().collect(); + let mut iter = cleaned.chars(); + iter.next(); + cleaned = upper + iter.as_str(); + } + + // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. + // The upstream Python only checks `isalnum` → period, but for our agent + // text we already may end in `!` `?` `.` etc. — treat any of those as OK. + let last = cleaned + .chars() + .next_back() + .expect("cleaned non-empty above"); + if !matches!(last, '.' | '!' | '?' | ';' | ':' | ',') { + cleaned.push('.'); + } + + // Count words before adding the cold-start prefix. + let word_count = cleaned.split_whitespace().count(); + + let max_frames = if word_count <= SHORT_PROMPT_WORD_THRESHOLD { + Some(SHORT_PROMPT_MAX_FRAMES) + } else { + // For everything ≥5 words, keep the upstream generation limits. + None + }; + + let mut final_text = String::with_capacity(cleaned.len() + 2); + final_text.push_str(". "); + final_text.push_str(&cleaned); + + Some(PreparedPrompt { + text: final_text, + max_frames, + }) +} + +/// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. +/// +/// Centralised so the invariant test below can assert that we **never** emit a +/// `frames_after_eos` override. Leaving the key unset preserves sherpa-onnx's +/// upstream default of 3. +fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { + prepared.max_frames.map(|mf| { + let mut h: HashMap = HashMap::with_capacity(1); + h.insert("max_frames".to_string(), serde_json::Value::from(mf)); + h + }) +} + +impl PocketTts { + /// Synthesise `text` with the given reference voice. + /// + /// `_lang` and `_steps` are accepted for API compatibility with the + /// previous Kokoro engine. Pocket TTS infers language from the input text + /// directly and is a one-step consistency model. Returns an empty buffer + /// for whitespace-only input. + pub fn synth_chunk( + &self, + text: &str, + lang: &str, + style: &VoiceStyle, + steps: usize, + ) -> Result, String> { + match self.synth_chunk_interruptible(text, lang, style, steps, || false)? { + SynthesisOutcome::Complete(samples) => Ok(samples), + SynthesisOutcome::Interrupted => Ok(Vec::new()), + } + } + + /// Synthesise one chunk while allowing the caller to stop in-flight model + /// generation. Interrupted output is discarded, so a partial waveform can + /// never be queued after a barge-in, voice change, or shutdown. + pub fn synth_chunk_interruptible( + &self, + text: &str, + _lang: &str, + style: &VoiceStyle, + _steps: usize, + is_interrupted: F, + ) -> Result + where + F: Fn() -> bool + Send + Sync + 'static, + { + // Mirror upstream pocket-tts prompt prep — without this short or + // unpunctuated inputs can cause the LM's EOS logit to never trip, + // producing up to 40 s of "monster breathing" garbage on the first + // utterance. See `prepare_pocket_prompt` for the full recipe. + let prepared = match prepare_pocket_prompt(text) { + Some(p) => p, + None => return Ok(SynthesisOutcome::Complete(Vec::new())), + }; + let is_interrupted = Arc::new(is_interrupted); + if is_interrupted() { + return Ok(SynthesisOutcome::Interrupted); + } + + // Per-call generation hints sherpa-onnx forwards to + // `offline-tts-pocket-impl.h`. We only override `max_frames`, and + // only for short padded prompts where we have a tight expectation + // on output length — that bounds the original runaway without + // disturbing the rest of the LM sampling envelope. See + // `prepare_pocket_prompt` docs for the regression history. + let extra = build_generation_extra(&prepared); + + let cfg = GenerationConfig { + num_steps: SYNTH_NUM_STEPS, + silence_scale: SYNTH_SILENCE_SCALE, + reference_audio: Some(style.samples.clone()), + reference_sample_rate: style.sample_rate, + extra, + // `speed` stays at its default: the Pocket impl never reads it + // (see the engine-contract note in the module docs). + ..Default::default() + }; + + let callback_interrupted = Arc::new(AtomicBool::new(false)); + let callback_flag = Arc::clone(&callback_interrupted); + let callback_predicate = Arc::clone(&is_interrupted); + let audio = self + .inner + .generate_with_config( + &prepared.text, + &cfg, + Some(move |_samples: &[f32], _progress: f32| { + let interrupted = callback_predicate(); + if interrupted { + callback_flag.store(true, Ordering::Release); + } + !interrupted + }), + ) + .ok_or_else(|| { + format!( + "Pocket TTS synthesis failed for text ({} chars)", + prepared.text.len() + ) + })?; + + let sample_rate = audio.sample_rate(); + if sample_rate != SAMPLE_RATE as i32 { + eprintln!( + "buzz-desktop: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ + (expected {SAMPLE_RATE}Hz); playback speed may be wrong" + ); + } + + if callback_interrupted.load(Ordering::Acquire) || is_interrupted() { + Ok(SynthesisOutcome::Interrupted) + } else { + Ok(SynthesisOutcome::Complete(audio.samples().to_vec())) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── prepare_pocket_prompt ──────────────────────────────────────────────── + + #[test] + fn prepare_prompt_returns_none_for_empty_input() { + assert!(prepare_pocket_prompt("").is_none()); + assert!(prepare_pocket_prompt(" ").is_none()); + assert!(prepare_pocket_prompt("\n\t ").is_none()); + } + + /// Production's cold-start prefix. + fn prompt_prefix() -> &'static str { + ". " + } + + #[test] + fn prepare_prompt_prefixes_and_capitalizes_one_word() { + // A bare lowercase word is prefixed, capitalized, terminated, and + // capped tightly enough to bound runaway generation. + let out = prepare_pocket_prompt("yep").expect("non-empty"); + assert_eq!(out.text, format!("{}Yep.", prompt_prefix())); + assert_eq!(out.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); + const { + assert!( + SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT, + "short cap must be tighter than the upstream default" + ); + } + } + + #[test] + fn prepare_prompt_preserves_existing_punctuation() { + let out = prepare_pocket_prompt("yes!").expect("non-empty"); + assert_eq!(out.text, format!("{}Yes!", prompt_prefix())); // exclamation kept + let out = prepare_pocket_prompt("really?").expect("non-empty"); + assert_eq!(out.text, format!("{}Really?", prompt_prefix())); + } + + #[test] + fn prepare_prompt_threshold_is_inclusive_at_four_words() { + // Both prompts are prefixed. Only the 4-word prompt gets a tight + // max_frames cap; the 5-word prompt keeps upstream generation limits. + let four = prepare_pocket_prompt("one two three four").expect("non-empty"); + assert_eq!( + four.text, + format!("{}One two three four.", prompt_prefix()), + "four-word input should get exactly the production prefix" + ); + assert_eq!(four.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); + + let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); + assert_eq!( + five.text, + format!("{}One two three four five.", prompt_prefix()), + "five-word input should also get the cold-start prefix" + ); + assert_eq!( + five.max_frames, None, + "long inputs must leave sherpa-onnx's max_frames default in place" + ); + } + + #[test] + fn prepare_prompt_prefixes_long_text_without_capping_it() { + let long = "This is a longer sentence that the model should handle just fine."; + let out = prepare_pocket_prompt(long).expect("non-empty"); + assert_eq!(out.text, format!("{}{}", prompt_prefix(), long)); + assert_eq!(out.max_frames, None); + assert!(out.text.ends_with('.')); + } + + #[test] + fn prepare_prompt_collapses_whitespace() { + let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); + // 3 words → short → prefixed. Interior whitespace collapsed. + assert_eq!(out.text, format!("{}Hello world friend.", prompt_prefix())); + } + + #[test] + fn prepare_prompt_does_not_double_capitalize_already_uppercase() { + let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); + assert_eq!(out.text, format!("{}HELLO there.", prompt_prefix())); + } + + #[test] + fn prepare_prompt_handles_non_ascii_first_letter() { + // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce + // mojibake. + let out = prepare_pocket_prompt("дa").expect("non-empty"); + assert!(out.text.contains("Дa.")); + } + + // ── build_generation_extra ─────────────────────────────────────────────── + // + // Short prompts override only `max_frames`; long prompts emit no extras. + // Every other knob remains at sherpa-onnx's documented default, notably + // `frames_after_eos = 3`. + + #[test] + fn build_extra_short_prompt_sets_only_max_frames() { + let prepared = prepare_pocket_prompt("yep").expect("non-empty"); + let extra = build_generation_extra(&prepared).expect("short prompts get extra"); + // Exactly one key — `max_frames` — and nothing else. + assert_eq!(extra.len(), 1, "extra has unexpected keys: {extra:?}"); + assert_eq!( + extra.get("max_frames"), + Some(&serde_json::Value::from(SHORT_PROMPT_MAX_FRAMES)) + ); + assert!( + !extra.contains_key("frames_after_eos"), + "frames_after_eos must never be set — upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT} is what we want" + ); + } + + #[test] + fn build_extra_long_prompt_is_none() { + // ≥5 words: no extras, so the upstream LM defaults remain authoritative. + let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); + assert_eq!( + build_generation_extra(&prepared), + None, + "long prompts must not override any LM knob" + ); + } + + #[test] + fn build_extra_never_lowers_frames_after_eos_for_any_word_count() { + // Sweep a range of prompt lengths and assert the `extra` map (when + // present) never carries a `frames_after_eos` override that's lower + // than the upstream sherpa-onnx default. Implemented as a structural + // check — we just never set the key — but worth a property test in + // case someone reintroduces the override in the future. + let prompts: &[&str] = &[ + "hi", + "hi there", + "yes please", + "one two three four", + "one two three four five", + "a slightly longer reply, hopefully fine", + "This is a multi-clause sentence. It has two parts.", + "really really really really really long prompt with lots of words just to be sure", + ]; + for &p in prompts { + let prepared = prepare_pocket_prompt(p).expect("non-empty"); + if let Some(extra) = build_generation_extra(&prepared) { + if let Some(v) = extra.get("frames_after_eos") { + let n = v.as_i64().expect("frames_after_eos should be int"); + assert!( + n >= SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT as i64, + "prompt {p:?} set frames_after_eos={n}, below upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT}" + ); + } + } + } + } + + #[test] + fn short_prompt_max_frames_is_below_upstream_default() { + // Sanity: the override only ever *lowers* the cap, never raises it. + const { + assert!(SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT); + } + // …and is still large enough for a one-to-four-word reply. At Mimi's + // 12.5 Hz frame rate, 100 frames = 8 s, which is roomy. + const { + assert!(SHORT_PROMPT_MAX_FRAMES >= 50, "would risk truncation"); + } + } +} diff --git a/crates/buzz-voice/tests/pocket_import_audio.rs b/crates/buzz-voice/tests/pocket_import_audio.rs new file mode 100644 index 0000000000..5936ee8d20 --- /dev/null +++ b/crates/buzz-voice/tests/pocket_import_audio.rs @@ -0,0 +1,360 @@ +use std::{ + fs, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread, + time::Duration, +}; + +use buzz_voice::{ + imported::{write_pcm16_wav, PcmStats, PocketVoiceLibrary}, + pocket::{ + load_text_to_speech, load_voice_style, SynthesisOutcome, DEFAULT_VOICE, SAMPLE_RATE, + VOICE_FILE_EXT, + }, +}; + +const PREVIEW_TEXT: &str = "This is an objective Pocket voice preview."; +const INTERRUPT_TEXT: &str = + "This longer Pocket voice sentence should stop before it produces audio that could be played."; + +fn required_path(name: &str) -> PathBuf { + std::env::var_os(name) + .map(PathBuf::from) + .unwrap_or_else(|| panic!("{name} must point to the required local test path")) +} + +fn checked_in_voice() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../desktop/src-tauri/resources/pocket-voices/marius.wav") +} + +fn evidence_dir() -> PathBuf { + std::env::var_os("BUZZ_VOICE_EVIDENCE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/buzz-voice-evidence") + }) +} + +fn synthesize(model_dir: &Path, voice_path: &Path, text: &str) -> (Vec, PcmStats) { + let engine = load_text_to_speech( + model_dir + .to_str() + .expect("Pocket model path must be valid UTF-8"), + ) + .expect("load Pocket model"); + let style = load_voice_style(voice_path).expect("load selected voice"); + let samples = engine + .synth_chunk(text, "en", &style, 1) + .expect("synthesize preview"); + let stats = PcmStats::analyze(&samples, SAMPLE_RATE); + assert!( + stats.is_non_silent(), + "generated PCM must be non-silent: {stats:?}" + ); + assert!( + stats.duration_seconds > 0.2, + "generated PCM is unexpectedly short: {stats:?}" + ); + (samples, stats) +} + +#[test] +#[ignore = "requires BUZZ_POCKET_MODEL_DIR and runs the installed Pocket ONNX model"] +fn objective_import_synthesis_delete_and_mary_fallback() { + let model_dir = required_path("BUZZ_POCKET_MODEL_DIR"); + let temp = tempfile::tempdir().expect("temporary voice workspace"); + let source = temp.path().join("Imported Marius.wav"); + fs::copy(checked_in_voice(), &source).expect("copy checked-in voice fixture"); + + let library_root = temp.path().join("library"); + let library = PocketVoiceLibrary::new(&library_root); + let imported = library.import_path(&source).expect("import valid WAV"); + assert_eq!( + library.find(&imported.key).expect("read selection"), + Some(imported.clone()) + ); + + drop(library); + let relaunched = PocketVoiceLibrary::new(&library_root); + let selected = relaunched + .find(&imported.key) + .expect("reload persisted selection") + .expect("selected imported voice survived relaunch"); + let imported_path = relaunched + .resolve_file(&selected) + .expect("resolve persisted imported voice"); + let (imported_pcm, imported_stats) = synthesize(&model_dir, &imported_path, PREVIEW_TEXT); + + let evidence = evidence_dir(); + fs::create_dir_all(&evidence).expect("create evidence directory"); + let imported_wav = evidence.join("imported-preview.wav"); + write_pcm16_wav(&imported_wav, &imported_pcm, SAMPLE_RATE) + .expect("write imported preview evidence"); + + let engine = load_text_to_speech( + model_dir + .to_str() + .expect("Pocket model path must be valid UTF-8"), + ) + .expect("load Pocket model for interruption"); + let style = load_voice_style(&imported_path).expect("load imported style for interruption"); + let interrupted = Arc::new(AtomicBool::new(false)); + let interrupt_worker = Arc::clone(&interrupted); + let interrupter = thread::spawn(move || { + thread::sleep(Duration::from_millis(25)); + interrupt_worker.store(true, Ordering::Release); + }); + let interrupt_check = Arc::clone(&interrupted); + let outcome = engine + .synth_chunk_interruptible(INTERRUPT_TEXT, "en", &style, 1, move || { + interrupt_check.load(Ordering::Acquire) + }) + .expect("interrupt synthesis"); + interrupter.join().expect("join interrupter"); + assert_eq!( + outcome, + SynthesisOutcome::Interrupted, + "interrupted synthesis must discard partial PCM" + ); + + assert!( + engine + .synth_chunk("", "en", &style, 1) + .expect("empty synthesis") + .is_empty(), + "empty input must not emit unintended audio" + ); + + relaunched + .delete(&imported.key) + .expect("delete imported voice"); + assert_eq!( + relaunched.find(&imported.key).expect("reload after delete"), + None + ); + + let mary_path = model_dir.join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}")); + assert_eq!( + mary_path.file_name().and_then(|name| name.to_str()), + Some("reference_sample.wav"), + "fallback must remain the deterministic Mary reference" + ); + let (mary_pcm, mary_stats) = synthesize(&model_dir, &mary_path, PREVIEW_TEXT); + let mary_wav = evidence.join("mary-fallback-preview.wav"); + write_pcm16_wav(&mary_wav, &mary_pcm, SAMPLE_RATE) + .expect("write Mary fallback preview evidence"); + + println!( + "{}", + serde_json::json!({ + "importedKey": imported.key, + "persistence": "reloaded", + "interruption": "partial PCM discarded", + "emptyInputSamples": 0, + "afterDelete": "pocket:mary", + "importedPreview": { + "path": imported_wav, + "samples": imported_stats.sample_count, + "sampleRate": imported_stats.sample_rate, + "durationSeconds": imported_stats.duration_seconds, + "peak": imported_stats.peak, + "rms": imported_stats.rms, + "nonSilentSamples": imported_stats.non_silent_samples, + }, + "maryFallbackPreview": { + "path": mary_wav, + "samples": mary_stats.sample_count, + "sampleRate": mary_stats.sample_rate, + "durationSeconds": mary_stats.duration_seconds, + "peak": mary_stats.peak, + "rms": mary_stats.rms, + "nonSilentSamples": mary_stats.non_silent_samples, + } + }) + ); +} + +#[cfg(target_os = "macos")] +mod blackhole { + use std::{ + sync::{Arc, Mutex}, + time::Instant, + }; + + use cpal::{ + self, + traits::{DeviceTrait, HostTrait, StreamTrait}, + SampleFormat, + }; + + use super::*; + + fn named_device( + devices: impl Iterator, + expected_name: &str, + ) -> cpal::Device { + devices + .filter_map(|device| { + device + .description() + .ok() + .map(|description| (device, description.name().to_string())) + }) + .find(|(_, name)| name == expected_name) + .map(|(device, _)| device) + .unwrap_or_else(|| panic!("audio device {expected_name:?} is not available")) + } + + fn capture_f32(device: &cpal::Device, captured: Arc>>) -> (cpal::Stream, u32) { + let supported = device + .default_input_config() + .expect("read BlackHole input configuration"); + assert_eq!( + supported.sample_format(), + SampleFormat::F32, + "BlackHole test expects Float32 input" + ); + let sample_rate = supported.sample_rate(); + let config: cpal::StreamConfig = supported.into(); + let stream = device + .build_input_stream( + &config, + move |data: &[f32], _| { + captured + .lock() + .unwrap_or_else(|error| error.into_inner()) + .extend_from_slice(data); + }, + |error| panic!("BlackHole capture failed: {error}"), + None, + ) + .expect("build BlackHole input stream"); + (stream, sample_rate) + } + + fn resample_linear(samples: &[f32], source_rate: u32, output_rate: u32) -> Vec { + let output_len = ((samples.len() as u64 * u64::from(output_rate) + + u64::from(source_rate) / 2) + / u64::from(source_rate)) as usize; + (0..output_len) + .map(|index| { + let source = index as f64 * f64::from(source_rate) / f64::from(output_rate); + let left = source.floor() as usize; + let fraction = (source - left as f64) as f32; + let a = samples[left.min(samples.len() - 1)]; + let b = samples[(left + 1).min(samples.len() - 1)]; + a + (b - a) * fraction + }) + .collect() + } + + #[test] + #[ignore = "requires BUZZ_POCKET_MODEL_DIR and BUZZ_VOICE_AUDIO_DEVICE=BlackHole 2ch"] + fn blackhole_playback_captures_generated_non_silent_pcm() { + let model_dir = required_path("BUZZ_POCKET_MODEL_DIR"); + let device_name = std::env::var("BUZZ_VOICE_AUDIO_DEVICE") + .expect("BUZZ_VOICE_AUDIO_DEVICE must name a BlackHole loopback device"); + let temp = tempfile::tempdir().expect("temporary voice workspace"); + let source = temp.path().join("Imported Marius.wav"); + fs::copy(checked_in_voice(), &source).expect("copy checked-in voice fixture"); + let library = PocketVoiceLibrary::new(temp.path().join("library")); + let imported = library.import_path(&source).expect("import voice"); + let selected = library + .find(&imported.key) + .expect("read selection") + .expect("selected imported voice"); + let voice_path = library + .resolve_file(&selected) + .expect("resolve imported voice"); + let (samples, generated_stats) = synthesize(&model_dir, &voice_path, PREVIEW_TEXT); + + let host = cpal::default_host(); + let input = named_device( + host.input_devices().expect("enumerate input devices"), + &device_name, + ); + let output = named_device( + host.output_devices().expect("enumerate output devices"), + &device_name, + ); + let captured = Arc::new(Mutex::new(Vec::new())); + let (input_stream, input_sample_rate) = capture_f32(&input, Arc::clone(&captured)); + input_stream.play().expect("start BlackHole capture"); + + let supported_output = output + .default_output_config() + .expect("read BlackHole output configuration"); + assert_eq!( + supported_output.sample_format(), + SampleFormat::F32, + "BlackHole test expects Float32 output" + ); + let output_sample_rate = supported_output.sample_rate(); + let output_config: cpal::StreamConfig = supported_output.into(); + let output_channels = usize::from(output_config.channels); + let playback = resample_linear(&samples, SAMPLE_RATE, output_sample_rate); + let playback_len = playback.len(); + let mut playback_index = 0usize; + let output_stream = output + .build_output_stream( + &output_config, + move |data: &mut [f32], _| { + for frame in data.chunks_mut(output_channels) { + let sample = playback.get(playback_index).copied().unwrap_or(0.0); + frame.fill(sample); + playback_index = playback_index.saturating_add(1); + } + }, + |error| panic!("BlackHole playback failed: {error}"), + None, + ) + .expect("build BlackHole output stream"); + output_stream.play().expect("start BlackHole playback"); + let started = Instant::now(); + let playback_duration = + Duration::from_secs_f64(playback_len as f64 / f64::from(output_sample_rate)); + while started.elapsed() < playback_duration { + thread::sleep(Duration::from_millis(20)); + } + thread::sleep(Duration::from_millis(250)); + drop(output_stream); + drop(input_stream); + + let captured_samples = captured + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + let captured_stats = PcmStats::analyze(&captured_samples, input_sample_rate); + assert!( + captured_stats.is_non_silent(), + "BlackHole must capture non-silent generated PCM: {captured_stats:?}" + ); + assert!( + captured_stats.non_silent_samples >= generated_stats.non_silent_samples / 4, + "captured signal is unexpectedly sparse: generated={generated_stats:?}, captured={captured_stats:?}" + ); + println!( + "{}", + serde_json::json!({ + "device": device_name, + "generated": { + "samples": generated_stats.sample_count, + "peak": generated_stats.peak, + "rms": generated_stats.rms, + }, + "captured": { + "samples": captured_stats.sample_count, + "sampleRate": captured_stats.sample_rate, + "peak": captured_stats.peak, + "rms": captured_stats.rms, + "nonSilentSamples": captured_stats.non_silent_samples, + } + }) + ); + } +} diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index 40dd5dd1b1..bbdccd6de3 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -48,6 +48,7 @@ export default defineConfig({ "**/activity-scope-label-screenshots.spec.ts", "**/welcome-agent-modal-screenshots.spec.ts", "**/local-archive-screenshots.spec.ts", + "**/voice-settings.spec.ts", "**/agent-readiness-screenshots.spec.ts", "**/agent-error-state-screenshots.spec.ts", "**/edit-agent.spec.ts", diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index 5f6f48eb14..7abf14b450 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -485,7 +485,10 @@ const overrides = new Map([ // +4 (1081 -> 1085): mesh recovery keeps one app-scoped state object beside // the embedded runtime and coordinator. Probe/re-arm logic lives in // mesh_llm/recovery.rs rather than growing AppState or command modules. - ["src-tauri/src/app_state.rs", 1085], + // Installation-global TTS settings, load diagnostics, and transition + // serialization live beside the huddle state they govern. + // Voice behavior remains split across the huddle modules. + ["src-tauri/src/app_state.rs", 1091], // multi-slot splitting + no-op suppression (#1309): the ReadStateManager // class grew from ~700 lines to ~1019 with the addition of // splitContextsIntoBudgetedSlots (pure fn + 5 tests), publishSplitSlots, diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 7dcf615c00..232aab147b 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -991,6 +991,7 @@ dependencies = [ name = "buzz-core" version = "0.1.0" dependencies = [ + "base64 0.22.1", "chrono", "hex", "hmac 0.13.0", @@ -1022,6 +1023,7 @@ dependencies = [ "buzz-media", "buzz-persona", "buzz-sdk", + "buzz-voice", "bytes", "bzip2 0.6.1", "chrono", @@ -1142,6 +1144,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "atomic-write-file", + "hex", + "serde", + "serde_json", + "sha2 0.11.0", + "sherpa-onnx", +] + [[package]] name = "by_address" version = "1.2.1" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index d689544688..d3c52235fd 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -89,6 +89,7 @@ buzz_core_pkg = { package = "buzz-core", path = "../../crates/buzz-core" } buzz_persona_pkg = { package = "buzz-persona", path = "../../crates/buzz-persona" } buzz_sdk_pkg = { package = "buzz-sdk", path = "../../crates/buzz-sdk" } buzz_agent_pkg = { package = "buzz-agent", path = "../../crates/buzz-agent" } +buzz_voice_pkg = { package = "buzz-voice", path = "../../crates/buzz-voice" } iroh = { version = "1.0.2", optional = true } mesh-llm-sdk = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-sdk", default-features = false, features = ["client", "serving"], optional = true } mesh-llm-host-runtime = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-host-runtime", default-features = false, features = ["dynamic-native-runtime"], optional = true } diff --git a/desktop/src-tauri/examples/pocket_quality_ab.rs b/desktop/src-tauri/examples/pocket_quality_ab.rs index 0c31f1c910..95f2e4815f 100644 --- a/desktop/src-tauri/examples/pocket_quality_ab.rs +++ b/desktop/src-tauri/examples/pocket_quality_ab.rs @@ -19,7 +19,7 @@ // standalone corpus generator deliberately does not call. #![allow(dead_code)] -#[path = "../src/huddle/pocket.rs"] +#[path = "../../../crates/buzz-voice/src/pocket.rs"] mod production_pocket; #[path = "../src/huddle/preprocessing.rs"] mod production_preprocessing; diff --git a/desktop/src-tauri/resources/pocket-voices/NOTICE.md b/desktop/src-tauri/resources/pocket-voices/NOTICE.md new file mode 100644 index 0000000000..ecbee2691d --- /dev/null +++ b/desktop/src-tauri/resources/pocket-voices/NOTICE.md @@ -0,0 +1,25 @@ +# Marius Pocket TTS reference voice + +`marius.wav` is an audio-identical copy of Kyutai's +`voice-donations/Selfie.wav`; only the filename changed. + +- Display name: Marius +- Source: https://huggingface.co/kyutai/tts-voices/blob/main/voice-donations/Selfie.wav +- Upstream revision: `323332d33f997de8394f24a193e1a76df720e01a` +- SHA-256: `076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16` +- Format: WAV PCM signed 16-bit little-endian, mono, 24 kHz, 10 seconds +- License: CC0 1.0 Universal + (https://creativecommons.org/publicdomain/zero/1.0/) +- Repository provenance: + https://huggingface.co/kyutai/tts-voices/blob/main/README.md +- Donation terms: + https://kyutai.org/next/legal/Terms%20of%20Use%20-%20Unmute%20Voice%20Donation%20Project%20v1.pdf + +The adult contributor submitted their own natural voice through Kyutai's +Unmute Voice Donation Project. Its terms expressly contemplate public reuse, +text-to-speech and voice-replica use, and synthetic speech generated in the +contributed voice. + +Neither Kyutai nor the contributor endorses Buzz. Buzz does not identify or +attempt to identify the contributor, and synthetic output must not be presented +as a genuine recording by them. diff --git a/desktop/src-tauri/resources/pocket-voices/marius.wav b/desktop/src-tauri/resources/pocket-voices/marius.wav new file mode 100644 index 0000000000..11ab460394 Binary files /dev/null and b/desktop/src-tauri/resources/pocket-voices/marius.wav differ diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index 94d162e620..1a2101a2d7 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -48,6 +48,9 @@ pub struct AppState { pub channel_templates_store_lock: Mutex<()>, pub managed_agent_processes: Mutex>, pub huddle_state: Mutex, + pub tts_settings: Mutex, + pub tts_settings_load_error: Mutex>, + pub tts_settings_transition: tokio::sync::Mutex<()>, /// Tauri app handle — stored after setup so huddle commands can emit /// `huddle-state-changed` events without needing the handle threaded /// through every call site. @@ -213,6 +216,9 @@ pub fn build_app_state() -> AppState { managed_agent_processes: Mutex::new(HashMap::new()), session_config_cache: Mutex::new(HashMap::new()), huddle_state: Mutex::new(HuddleState::default()), + tts_settings: Mutex::new(Default::default()), + tts_settings_load_error: Mutex::new(None), + tts_settings_transition: tokio::sync::Mutex::new(()), app_handle: Mutex::new(None), audio_output_device: Mutex::new(None), media_proxy_port: AtomicU16::new(0), diff --git a/desktop/src-tauri/src/huddle/agent_tts_routing_tests.rs b/desktop/src-tauri/src/huddle/agent_tts_routing_tests.rs new file mode 100644 index 0000000000..c6ce686853 --- /dev/null +++ b/desktop/src-tauri/src/huddle/agent_tts_routing_tests.rs @@ -0,0 +1,26 @@ +use super::{enqueue_agent_tts_text, normalize_agent_tts_text, MAX_TTS_TEXT_LEN}; + +#[tokio::test] +async fn assistant_plain_text_routes_unchanged_into_voice_pipeline_boundary() { + let (sender, receiver) = std::sync::mpsc::channel(); + let text = "A newly submitted assistant reply.".to_string(); + + enqueue_agent_tts_text(text.clone(), move |queued| { + sender.send(queued).map_err(|error| error.to_string()) + }) + .await + .expect("route assistant text"); + + assert_eq!(receiver.recv().expect("queued text"), text); +} + +#[test] +fn assistant_text_truncation_is_unicode_safe_before_voice_routing() { + let input = "🦀".repeat(MAX_TTS_TEXT_LEN + 1); + let output = normalize_agent_tts_text(input); + assert_eq!( + output.chars().count(), + MAX_TTS_TEXT_LEN + "... message truncated.".chars().count() + ); + assert!(output.ends_with("... message truncated.")); +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index a815bf2d06..08802736c3 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -37,6 +37,8 @@ pub mod state; pub mod stt; pub mod transcription; pub mod tts; +pub mod tts_settings; +mod tts_voice_import; pub mod wire; // ── Shared utilities ────────────────────────────────────────────────────────── @@ -63,6 +65,7 @@ pub(super) fn drain_until_shutdown( pub use state::{HuddleJoinInfo, HuddlePhase, HuddleState, VoiceInputMode}; pub use transcription::{set_huddle_transcription_enabled, start_stt_pipeline}; +pub use tts_settings::set_tts_enabled; // ── Imports ─────────────────────────────────────────────────────────────────── @@ -817,53 +820,31 @@ pub fn get_model_status(_state: State<'_, AppState>) -> Result) -> Result<(), String> { - let old_pipeline = { - let mut hs = state.huddle()?; - hs.tts_enabled = enabled; - if !enabled { - hs.tts_pipeline.take() // Take out of lock. - } else { - None - } - }; - // Shut down outside the lock — thread join happens here. - if let Some(ref pipeline) = old_pipeline { - pipeline.shutdown(); - } - drop(old_pipeline); - - if enabled { - // Re-start TTS pipeline if models are available and huddle is active. - let phase = { - let hs = state.huddle()?; - hs.phase.clone() - }; - if matches!(phase, HuddlePhase::Connected | HuddlePhase::Active) { - if let Err(e) = maybe_start_tts_pipeline(&state).await { - eprintln!("buzz-desktop: TTS pipeline restart failed: {e}"); - } - } - } - - Ok(()) -} - /// Speak an agent message via TTS. /// /// Maximum text length accepted for TTS synthesis. /// ~2000 chars ≈ 1–2 minutes of speech. Longer messages are truncated. const MAX_TTS_TEXT_LEN: usize = 2000; +fn normalize_agent_tts_text(text: String) -> String { + if text.chars().count() > MAX_TTS_TEXT_LEN { + let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect(); + truncated.push_str("... message truncated."); + truncated + } else { + text + } +} + +async fn enqueue_agent_tts_text(text: String, enqueue: F) -> Result<(), String> +where + F: FnOnce(String) -> Result<(), String> + Send + 'static, +{ + tokio::task::spawn_blocking(move || enqueue(text)) + .await + .map_err(|error| format!("TTS enqueue task failed: {error}"))? +} + /// Called by the WebView when it receives an incoming agent kind:9 message. /// Lazily starts the TTS pipeline if models are ready but the pipeline hasn't /// been created yet (e.g. models finished downloading after huddle started). @@ -873,13 +854,7 @@ const MAX_TTS_TEXT_LEN: usize = 2000; pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Result<(), String> { // Truncate oversized messages — agents shouldn't monologue in a voice huddle. // Use char count (not byte length) to avoid panicking on multi-byte UTF-8. - let text = if text.chars().count() > MAX_TTS_TEXT_LEN { - let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect(); - truncated.push_str("... message truncated."); - truncated - } else { - text - }; + let text = normalize_agent_tts_text(text); let needs_pipeline = { let hs = state.huddle()?; @@ -895,15 +870,30 @@ pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Re } } - let hs = state.huddle()?; - if hs.tts_enabled { - if let Some(ref pipeline) = hs.tts_pipeline { - pipeline.speak(text)?; - } - } - Ok(()) + let sender = { + let hs = state.huddle()?; + hs.tts_enabled + .then(|| { + hs.tts_pipeline + .as_ref() + .map(|pipeline| pipeline.text_sender()) + }) + .flatten() + }; + let Some(sender) = sender else { + return Ok(()); + }; + enqueue_agent_tts_text(text, move |text| { + sender + .send(text) + .map_err(|error| format!("TTS queue closed while waiting to enqueue: {error}")) + }) + .await } +#[cfg(test)] +mod agent_tts_routing_tests; + /// Add an agent to the active huddle. /// /// Steps: diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 169ddf66c0..e9b4e02a2c 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -24,6 +24,9 @@ use std::sync::{Arc, Mutex, OnceLock}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +#[path = "models_voice_upgrade.rs"] +mod voice_upgrade; + // ── Integrity verification ──────────────────────────────────────────────────── // // All model artifacts are verified against pinned SHA-256 hashes before @@ -77,6 +80,7 @@ const TTS_FILE_HASHES: &[(&str, &str)] = &[ ("token_scores.json", "5be2f278caf9b9800741f0fd82bff677f4943ec764c356f907213434b622d958"), ("LICENSE", "fe7b4ce83b8381cc5b216bbb4af73c570688d1b819c73bbaed8ca401f4677cd6"), ("reference_sample.wav", "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f"), + ("marius.wav", "076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16"), ]; // ── Model versioning ────────────────────────────────────────────────────────── @@ -100,7 +104,7 @@ const STT_MODEL_VERSION: &str = "2"; /// reason explicit and skips the failing-then-re-fetching transient state. /// Bumped "2" → "3" for the int8 → fp32 model swap (see `POCKET_HF_BASE`): /// existing int8 installs must re-download the suffixless fp32 sessions. -const TTS_MODEL_VERSION: &str = "3"; +const TTS_MODEL_VERSION: &str = "4"; /// Filename for the version manifest written alongside model files. const MANIFEST_FILENAME: &str = ".buzz-model-manifest"; @@ -194,6 +198,14 @@ https://datashare.ed.ac.uk/handle/10283/3443 (CC-BY-4.0). Recording enhancement (denoise/dereverb) by ai-coustics: https://ai-coustics.com/ +Bundled reference voice (marius.wav): +\"Marius\", an audio-identical copy of Kyutai's +`voice-donations/Selfie.wav`, distributed under CC0 1.0 Universal: +https://huggingface.co/kyutai/tts-voices/blob/main/voice-donations/Selfie.wav +https://creativecommons.org/publicdomain/zero/1.0/ +The contributor submitted their own voice under Kyutai's Unmute Voice Donation +Project terms. Neither Kyutai nor the contributor endorses Buzz. + Buzz ships all ONNX/model artifacts and the reference voice WAV unmodified, renamed only by placement in the local model directory. @@ -212,6 +224,7 @@ const TTS_EXPECTED_FILES: &[&str] = &[ "token_scores.json", "LICENSE", "reference_sample.wav", + "marius.wav", TTS_LICENSE_FILE_NAME, ]; @@ -640,6 +653,9 @@ impl ModelManager { /// Start a background Pocket TTS download (~189 MB). No-op if already ready or downloading. pub fn start_tts_download(&self, http_client: reqwest::Client) { + if let Err(error) = voice_upgrade::install_marius_into_v3_model(&self.models_dir) { + eprintln!("buzz-desktop: could not upgrade existing Pocket voices in place: {error}"); + } let manager = self.clone(); self.tts.start_download( &self.models_dir, @@ -757,7 +773,7 @@ impl ModelManager { /// - five ONNX sessions (Pocket TTS + Mimi codec) /// - `vocab.json` / `token_scores.json` for sherpa-onnx text conditioning /// - upstream `LICENSE` plus Buzz's `MODEL_LICENSE.txt` attribution sidecar - /// - `reference_sample.wav` as the bundled default voice + /// - `reference_sample.wav` and embedded `marius.wav` reference voices /// /// Files are written to a temp directory first, then moved atomically. async fn download_tts_model(&self, http_client: reqwest::Client) -> Result<(), String> { @@ -845,6 +861,12 @@ impl ModelManager { tokio::fs::write(temp_dir.join(TTS_LICENSE_FILE_NAME), TTS_LICENSE_TEXT) .await .map_err(|e| format!("write TTS model license sidecar: {e}"))?; + tokio::fs::write( + temp_dir.join("marius.wav"), + voice_upgrade::POCKET_MARIUS_WAV, + ) + .await + .map_err(|e| format!("install bundled Marius voice: {e}"))?; self.tts.set_status(ModelStatus::Downloading { progress_percent: 90, diff --git a/desktop/src-tauri/src/huddle/models_voice_upgrade.rs b/desktop/src-tauri/src/huddle/models_voice_upgrade.rs new file mode 100644 index 0000000000..28c6d26da8 --- /dev/null +++ b/desktop/src-tauri/src/huddle/models_voice_upgrade.rs @@ -0,0 +1,73 @@ +use super::*; + +pub(super) const POCKET_MARIUS_WAV: &[u8] = + include_bytes!("../../resources/pocket-voices/marius.wav"); +const PRE_MARIUS_TTS_MODEL_VERSION: &str = "3"; + +/// Add Marius to an otherwise-ready v3 install without re-downloading models. +/// +/// The manifest is written last, so interruption leaves v3 intact and the next +/// launch retries. +pub(super) fn install_marius_into_v3_model(models_dir: &Path) -> Result<(), String> { + let model_dir = models_dir.join(TTS_MODEL_DIR_NAME); + let manifest_path = model_dir.join(MANIFEST_FILENAME); + let version = match std::fs::read_to_string(&manifest_path) { + Ok(version) => version, + Err(_) => return Ok(()), + }; + if version.trim() != PRE_MARIUS_TTS_MODEL_VERSION { + return Ok(()); + } + if !TTS_EXPECTED_FILES + .iter() + .filter(|filename| **filename != "marius.wav") + .all(|filename| model_dir.join(filename).is_file()) + { + return Ok(()); + } + + std::fs::write(model_dir.join("marius.wav"), POCKET_MARIUS_WAV) + .map_err(|error| format!("write Marius voice: {error}"))?; + std::fs::write(model_dir.join(TTS_LICENSE_FILE_NAME), TTS_LICENSE_TEXT) + .map_err(|error| format!("update Pocket voice notice: {error}"))?; + std::fs::write(manifest_path, TTS_MODEL_VERSION) + .map_err(|error| format!("update Pocket model manifest: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn v3_install_adds_marius_without_redownloading_models() { + let temp = tempfile::tempdir().expect("tempdir"); + let model_dir = temp.path().join(TTS_MODEL_DIR_NAME); + std::fs::create_dir_all(&model_dir).expect("create model dir"); + for file in TTS_EXPECTED_FILES + .iter() + .filter(|filename| **filename != "marius.wav") + { + std::fs::write(model_dir.join(file), b"existing").expect("write prior file"); + } + std::fs::write( + model_dir.join(MANIFEST_FILENAME), + PRE_MARIUS_TTS_MODEL_VERSION, + ) + .expect("write prior manifest"); + + install_marius_into_v3_model(temp.path()).expect("in-place upgrade"); + + assert_eq!( + std::fs::read(model_dir.join("marius.wav")).expect("Marius installed"), + POCKET_MARIUS_WAV + ); + assert_eq!( + std::fs::read_to_string(model_dir.join(MANIFEST_FILENAME)).expect("updated manifest"), + TTS_MODEL_VERSION + ); + assert!( + ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION) + .is_ready(temp.path()) + ); + } +} diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs index ceccedd8b6..d58f9d7e18 100644 --- a/desktop/src-tauri/src/huddle/pipeline.rs +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -16,7 +16,7 @@ use crate::events; use super::models; use super::relay_api::{self, fetch_channel_members, parse_channel_uuid}; -use super::state::{HuddlePhase, VoiceInputMode}; +use super::state::{HuddlePhase, HuddleState, VoiceInputMode}; use super::stt; use super::tts; @@ -192,6 +192,38 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result return Ok(false), }; + // Avoid resolving and hashing imported voice files on every hot-start poll + // when TTS is already disabled or running. The guarded claim below repeats + // these checks after the fallible work to close the race. + { + let huddle = state.huddle()?; + if huddle.tts_pipeline.is_some() || !huddle.tts_enabled { + return Ok(false); + } + } + + // Resolve all fallible construction inputs before claiming the sentinel so + // an unreadable optional voice registry cannot wedge future start attempts. + let output_device = state + .audio_output_device + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + let app = state + .app_handle + .lock() + .map_err(|error| format!("app handle lock poisoned: {error}"))? + .clone(); + let voice_preferences = state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}")) + .map(|settings| settings.voice_preferences.clone())?; + let initial_voice = match app { + Some(app) => super::tts_settings::pocket_voice_reference(&app, &voice_preferences)?, + None => super::tts_settings::bundled_pocket_voice_reference(&voice_preferences), + }; + // Atomically check preconditions and claim the construction slot. // The sentinel prevents a second caller from starting construction // while we're building outside the lock. @@ -211,13 +243,14 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result Result Result { + let mut huddle = state.huddle()?; + huddle.tts_starting.store(false, Ordering::Release); + if !huddle.tts_enabled + || !matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active) + || huddle.tts_pipeline.is_some() { - let mut hs = state.huddle()?; - hs.tts_starting.store(false, Ordering::Release); - // Phase check: huddle may have been torn down during construction. - if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { - return Ok(false); - } - // Final check: another path may have created a pipeline while we were constructing. - if hs.tts_pipeline.is_some() { - return Ok(false); - } - hs.tts_pipeline = Some(pipeline); + return Ok(false); } - + let app = state + .app_handle + .lock() + .map_err(|error| format!("app handle lock poisoned: {error}"))? + .clone(); + let preferences = state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}")) + .map(|settings| settings.voice_preferences.clone())?; + let voice = match app { + Some(app) => super::tts_settings::pocket_voice_reference(&app, &preferences)?, + None => super::tts_settings::bundled_pocket_voice_reference(&preferences), + }; + publish(&voice, &mut huddle); Ok(true) } @@ -357,3 +415,98 @@ pub(crate) fn spawn_transcription_task( } }); } + +#[cfg(test)] +mod tts_start_race_tests { + use std::sync::{atomic::Ordering, Arc, Barrier, Mutex}; + + use crate::app_state::build_app_state; + + use super::{finalize_tts_pipeline_start, HuddlePhase}; + + #[test] + fn construction_reconciles_a_voice_selected_while_starting() { + let state = Arc::new(build_app_state()); + { + let mut huddle = state.huddle().expect("huddle state"); + huddle.phase = HuddlePhase::Active; + huddle.tts_enabled = true; + huddle.tts_starting.store(true, Ordering::Release); + } + + let constructed = Arc::new(Barrier::new(2)); + let publish = Arc::new(Barrier::new(2)); + let selected_voice = Arc::new(Mutex::new(None)); + let worker_state = Arc::clone(&state); + let worker_constructed = Arc::clone(&constructed); + let worker_publish = Arc::clone(&publish); + let worker_voice = Arc::clone(&selected_voice); + let worker = std::thread::spawn(move || { + worker_constructed.wait(); + worker_publish.wait(); + finalize_tts_pipeline_start(&worker_state, |voice, _| { + *worker_voice.lock().expect("selected voice") = Some(voice.to_string()); + }) + }); + + constructed.wait(); + assert!(state + .huddle() + .expect("huddle state") + .tts_starting + .load(Ordering::Acquire)); + state + .tts_settings + .lock() + .expect("text-to-speech settings") + .voice_preferences = vec!["pocket:marius".to_string()]; + publish.wait(); + + assert!(worker.join().expect("starter thread").expect("finalize")); + assert_eq!( + *selected_voice.lock().expect("selected voice"), + Some("marius".to_string()) + ); + } + + #[test] + fn construction_is_discarded_when_disabled_while_starting() { + let state = Arc::new(build_app_state()); + { + let mut huddle = state.huddle().expect("huddle state"); + huddle.phase = HuddlePhase::Active; + huddle.tts_enabled = true; + huddle.tts_starting.store(true, Ordering::Release); + } + + let constructed = Arc::new(Barrier::new(2)); + let publish = Arc::new(Barrier::new(2)); + let did_publish = Arc::new(Mutex::new(false)); + let worker_state = Arc::clone(&state); + let worker_constructed = Arc::clone(&constructed); + let worker_publish = Arc::clone(&publish); + let worker_did_publish = Arc::clone(&did_publish); + let worker = std::thread::spawn(move || { + worker_constructed.wait(); + worker_publish.wait(); + finalize_tts_pipeline_start(&worker_state, |_, _| { + *worker_did_publish.lock().expect("publish flag") = true; + }) + }); + + constructed.wait(); + { + let mut huddle = state.huddle().expect("huddle state"); + huddle.tts_enabled = false; + } + publish.wait(); + + assert!(!worker.join().expect("starter thread").expect("finalize")); + assert!(!*did_publish.lock().expect("publish flag")); + assert!(!state + .huddle() + .expect("huddle state") + .tts_starting + .load(Ordering::Acquire)); + } +} diff --git a/desktop/src-tauri/src/huddle/pocket.rs b/desktop/src-tauri/src/huddle/pocket.rs index ee1faf928a..8e19781eeb 100644 --- a/desktop/src-tauri/src/huddle/pocket.rs +++ b/desktop/src-tauri/src/huddle/pocket.rs @@ -1,654 +1,3 @@ -//! Pocket TTS engine wrapper around sherpa-onnx's `OfflineTts`. -//! -//! Pocket TTS is a small (~473 MB fp32 ONNX) zero-shot voice-cloning TTS -//! model from Kyutai. It runs quickly on CPU via sherpa-onnx, replacing the -//! previous Kokoro-82M engine that also required an espeak-free but -//! lexicon-heavy G2P pipeline (Misaki + CMUdict). -//! -//! Full-precision fp32 sessions, not the ~189 MB int8 quantization we -//! originally shipped: a direct same-runtime A/B (k2-fsa/sherpa-onnx#3172) -//! found the int8 ONNX export audibly degraded output quality, and fp32 -//! "significantly improved quality even at 1 step". -//! -//! ## Attribution -//! -//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. -//! arXiv:2509.06926. Original repository: . -//! Licensed CC-BY-4.0. -//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. -//! - **ONNX export**: KevinAHM — -//! . CC-BY-4.0. -//! - **sherpa-onnx repackage**: csukuangfj / k2-fsa — -//! . -//! Repackages KevinAHM's export with the file layout sherpa-onnx's -//! `OfflineTtsPocketModelConfig` expects. CC-BY-4.0. -//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary -//! (f, conversation)" preset from the Kyutai TTS demo -//! (), which maps to `vctk/p333_023_enhanced.wav` -//! in . CC-BY-4.0, base recording -//! from the VCTK corpus, enhanced by ai-coustics. -//! -//! Buzz ships these files unmodified; see the on-disk `MODEL_LICENSE.txt` -//! sidecar written by `huddle::models` during install for the canonical -//! CC-BY-4.0 §3(a)(1) attribution block. -//! -//! ## Engine-module contract (see `huddle::tts`) -//! -//! `pocket.rs` exposes a fixed surface used by `tts.rs`. Mirroring this -//! contract is what lets the TTS pipeline stay engine-agnostic: -//! -//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. -//! - `DEFAULT_VOICE: &str` — default voice name (without extension). -//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. -//! - `load_text_to_speech(model_dir)` → `Result` -//! - `load_voice_style(path)` → `Result` -//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` -//! → `Result, String>` -//! -//! `lang` and `steps` are accepted for API compatibility with the previous -//! Kokoro engine but are unused — Pocket TTS does its own language ID from -//! the input text and is not a diffusion model (consistency LM, one step). -//! There is no speed knob: sherpa-onnx's `GenerationConfig.speed` is only -//! read by some model families (vits), never by the Pocket impl -//! (`offline-tts-pocket-impl.h` — zero references), and upstream pocket-tts -//! has no speed parameter either. +//! Compatibility re-export for the shared Pocket TTS engine. -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; - -// ── Engine-module contract: public consts ───────────────────────────────────── - -/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, -/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. -pub const SAMPLE_RATE: u32 = 24_000; - -/// Name (without extension) of the bundled reference voice. The model directory -/// is expected to contain `.` after install. -pub const DEFAULT_VOICE: &str = "reference_sample"; - -/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the -/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, -/// not a precomputed embedding, so the extension change is honest. -pub const VOICE_FILE_EXT: &str = "wav"; - -// ── Tuning ──────────────────────────────────────────────────────────────────── - -/// Single-threaded ONNX execution for predictable CPU contention with the STT -/// pipeline. Matches `STT_NUM_THREADS` in `stt.rs`; raise only if a benchmark -/// argues for it. -const TTS_NUM_THREADS: i32 = 1; - -/// LRU cache size for cloned voice embeddings inside the sherpa-onnx engine. -/// We bind to one voice per pipeline today, but the upstream example uses 16 -/// and the cost is negligible — keep room for future multi-voice support. -const VOICE_EMBEDDING_CACHE_CAPACITY: i32 = 16; - -/// Pocket TTS is a consistency-based LM. Generation quality saturates at one -/// denoising step — the upstream `GenerationConfig` default of 5 multiplies -/// synthesis time by ~5× with no audible benefit on this model. -const SYNTH_NUM_STEPS: i32 = 1; - -/// Leave the generated audio's silences untouched (1.0 is the identity). -/// -/// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding -/// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and -/// multiplies its length by this factor. The previous value of 0.0 — set -/// under the mistaken belief it disabled lead-in/lead-out padding — deleted -/// every natural pause inside an utterance: clause breaks, breaths, the gap -/// after a comma. Words slammed together and endings cut abruptly. The -/// reference Pocket TTS pipeline does not post-process silence at all; -/// 1.0 restores parity. -const SYNTH_SILENCE_SCALE: f32 = 1.0; - -/// sherpa-onnx upstream default for `max_frames` (LM steps), in -/// `offline-tts-pocket-impl.h:Generate`. 500 steps ≈ 40 s of audio at the -/// Mimi 12.5 Hz frame rate. Referenced only by the regression test below; -/// production code path never raises (or even reads) this value — we just -/// leave sherpa-onnx's own default in place by not setting the override. -#[cfg(test)] -const SHERPA_ONNX_MAX_FRAMES_DEFAULT: i32 = 500; - -/// Tight `max_frames` we ask for on short, padded prompts to bound the -/// original "monster breathing" runaway. 100 LM steps ≈ 8 s of audio — -/// roomy for any one-to-four-word utterance the user is likely to elicit -/// while still well short of the 40 s upstream default. Chosen with slack so -/// we never *truncate* a legitimate short reply. -const SHORT_PROMPT_MAX_FRAMES: i32 = 100; - -/// Word-count threshold (inclusive) below which we pad the prompt with -/// leading spaces and cap `max_frames` tighter than the upstream default. -/// Matches upstream `pocket_tts.models.tts_model.prepare_text_prompt`. Above -/// this threshold we leave sherpa-onnx's own defaults in place — overriding -/// them caused the "first 'yep' is just static" regression seen on -/// 2026-05-18, where dropping `frames_after_eos` below the upstream default -/// of 3 clipped the leading audio of multi-clause sentences. -const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; - -/// Number of leading spaces prepended to short prompts. The upstream Python -/// uses exactly 8 — keep parity rather than tuning blindly. -/// -/// This is upstream's *only* mitigation for the FlowLM cold-start smear on -/// short utterances (kyutai-labs/pocket-tts #91, #70): the autoregressive -/// generation has a 2–3 step "settle" period where the first phoneme can be -/// smeared. A previous revision added a sacrificial `". . "` prefix plus an -/// amplitude-threshold trim to strip the rendered prefix from the output — -/// but the trim's absolute threshold (0.02 against raw peaks of ~0.076) sat -/// in soft-onset territory and could eat real word starts, and its tuning -/// was calibrated against `silence_scale = 0.0` audio. Deleted in favour of -/// upstream parity: accept the occasional smeared first syllable rather -/// than risk trimming real speech. -const SHORT_PROMPT_PAD_SPACES: usize = 8; - -/// sherpa-onnx's documented `frames_after_eos` default. We deliberately do -/// *not* override this knob — the previous attempt to bump it for short -/// inputs and lower it for long inputs lowered it below the upstream default -/// of 3, which clipped the leading audio of multi-clause sentences (the -/// "first 'yep' is static" regression). The constant exists only for the -/// regression test below. Source: `offline-tts-pocket-impl.h:Generate`. -#[cfg(test)] -const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; - -// ── ONNX file names (five Pocket TTS sessions plus two JSON tables) ─────────── - -const FILE_LM_MAIN: &str = "lm_main.onnx"; -const FILE_LM_FLOW: &str = "lm_flow.onnx"; -const FILE_ENCODER: &str = "encoder.onnx"; -const FILE_DECODER: &str = "decoder.onnx"; -const FILE_TEXT_COND: &str = "text_conditioner.onnx"; -const FILE_VOCAB: &str = "vocab.json"; -const FILE_TOKEN_SCORES: &str = "token_scores.json"; - -// ── Voice style ─────────────────────────────────────────────────────────────── - -/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. -/// -/// Pocket TTS takes a reference waveform per generation call (not a -/// precomputed style embedding), so we keep the samples in memory and clone -/// the small `Vec` into each `GenerationConfig` rather than re-reading the -/// WAV from disk on every sentence. -#[derive(Debug, Clone)] -pub struct VoiceStyle { - samples: Vec, - sample_rate: i32, -} - -/// Load a reference voice WAV from disk. -/// -/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS -/// resamples internally using `reference_sample_rate`. The bundled -/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. -pub fn load_voice_style(path: &Path) -> Result { - let path_str = path - .to_str() - .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; - let wave = Wave::read(path_str) - .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; - let samples = wave.samples().to_vec(); - if samples.is_empty() { - return Err(format!("voice WAV is empty: {}", path.display())); - } - Ok(VoiceStyle { - samples, - sample_rate: wave.sample_rate(), - }) -} - -// ── Engine ──────────────────────────────────────────────────────────────────── - -/// Pocket TTS engine handle. Cheap to construct (one `OfflineTts::create` -/// call). Owned by the TTS worker thread for the lifetime of a huddle session. -/// -/// `OfflineTts` does not implement `Debug`, so we don't derive it here — the -/// pipeline only needs to move the engine into the worker thread and call -/// `synth_chunk` on it, never to print it. -pub struct PocketTts { - inner: OfflineTts, -} - -/// Build the Pocket TTS engine from the model directory installed by -/// `huddle::models`. Returns `Err` if any expected ONNX or JSON file is -/// missing — readiness is normally enforced by `is_tts_ready` upstream, but -/// the check is repeated here so a manually-modified model dir produces a -/// clear error string instead of an opaque sherpa-onnx `None`. -pub fn load_text_to_speech(model_dir: &str) -> Result { - let dir = PathBuf::from(model_dir); - for name in [ - FILE_LM_MAIN, - FILE_LM_FLOW, - FILE_ENCODER, - FILE_DECODER, - FILE_TEXT_COND, - FILE_VOCAB, - FILE_TOKEN_SCORES, - ] { - let p = dir.join(name); - if !p.is_file() { - return Err(format!("missing Pocket TTS file: {}", p.display())); - } - } - - let to_str = |name: &str| -> String { dir.join(name).to_string_lossy().into_owned() }; - - // Build the config by mutating defaults — mirrors `stt.rs` and stays - // resilient if sherpa-onnx adds unrelated model-family fields. - let mut cfg = OfflineTtsConfig::default(); - cfg.model.pocket.lm_main = Some(to_str(FILE_LM_MAIN)); - cfg.model.pocket.lm_flow = Some(to_str(FILE_LM_FLOW)); - cfg.model.pocket.encoder = Some(to_str(FILE_ENCODER)); - cfg.model.pocket.decoder = Some(to_str(FILE_DECODER)); - cfg.model.pocket.text_conditioner = Some(to_str(FILE_TEXT_COND)); - cfg.model.pocket.vocab_json = Some(to_str(FILE_VOCAB)); - cfg.model.pocket.token_scores_json = Some(to_str(FILE_TOKEN_SCORES)); - cfg.model.pocket.voice_embedding_cache_capacity = VOICE_EMBEDDING_CACHE_CAPACITY; - cfg.model.num_threads = TTS_NUM_THREADS; - // Explicit — defaults are not part of the API contract, and noisy debug - // logging in release builds would be expensive on every synthesized chunk. - cfg.model.debug = false; - - let inner = OfflineTts::create(&cfg) - .ok_or_else(|| "OfflineTts::create returned None for Pocket TTS".to_string())?; - Ok(PocketTts { inner }) -} - -// ── Prompt preparation ──────────────────────────────────────────────────────── - -/// Result of [`prepare_pocket_prompt`]: a synthesizer-ready prompt plus the -/// per-call generation overrides derived from the original text. -/// -/// `None` for either override means "leave sherpa-onnx's documented default -/// in place". The pipeline only sets `max_frames` (and only for short -/// padded inputs) so it can bound the original "monster breathing" runaway -/// without disturbing the rest of the LM sampling envelope. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct PreparedPrompt { - /// Text to hand to `OfflineTts::generate_with_config`. Capitalized, - /// punctuation-terminated, and (for short inputs) left-padded with - /// spaces — upstream's mitigation for the FlowLM cold-start smear. - pub text: String, - /// Value to pass via `GenerationConfig.extra["max_frames"]`, or `None` to - /// keep the upstream default of 500 LM steps. We only override on short - /// padded prompts where we have a tight expectation on output length. - pub max_frames: Option, -} - -/// Mirror of the *text-preparation* half of upstream -/// `pocket_tts.models.tts_model.prepare_text_prompt`. Sherpa-onnx's C++ -/// Pocket TTS impl does not run these preparation steps, so short / -/// unpunctuated / lowercase inputs can trigger up to 40 s of runaway -/// generation when the EOS logit never crosses its threshold. We replicate -/// the upstream Python recipe here: -/// -/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but -/// cheap to re-check after sentence splitting). -/// 2. Capitalize the first letter. -/// 3. Append `.` if the text doesn't end in punctuation. -/// 4. If fewer than five words, prepend `SHORT_PROMPT_PAD_SPACES` spaces -/// (upstream's cold-start mitigation — see the constant's docstring) and -/// return a tight [`SHORT_PROMPT_MAX_FRAMES`] cap so the LM can't run -/// away if EOS still doesn't fire. -/// -/// We do **not** override `frames_after_eos` — sherpa-onnx's default of 3 -/// is what we want. An earlier version set it to 1 on long inputs, which -/// clipped the leading audio of multi-clause sentences ("first 'yep' is -/// just static" regression). Tests `prepare_prompt_never_lowers_frames_…` -/// lock this in. -/// -/// Returns `None` only if the input is empty after trimming — caller should -/// skip synthesis in that case. -pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { - let trimmed = input.trim(); - if trimmed.is_empty() { - return None; - } - - // Collapse stray double-spaces / embedded newlines that may slip past - // `preprocess_for_tts` when sentences are spliced back together. - let mut cleaned = String::with_capacity(trimmed.len()); - let mut last_was_space = false; - for ch in trimmed.chars() { - let is_ws = ch.is_whitespace(); - if is_ws { - if !last_was_space { - cleaned.push(' '); - } - last_was_space = true; - } else { - cleaned.push(ch); - last_was_space = false; - } - } - - // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). - let first = cleaned.chars().next().expect("cleaned non-empty above"); - if first.is_lowercase() { - let upper: String = first.to_uppercase().collect(); - let mut iter = cleaned.chars(); - iter.next(); - cleaned = upper + iter.as_str(); - } - - // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. - // The upstream Python only checks `isalnum` → period, but for our agent - // text we already may end in `!` `?` `.` etc. — treat any of those as OK. - let last = cleaned - .chars() - .next_back() - .expect("cleaned non-empty above"); - if !matches!(last, '.' | '!' | '?' | ';' | ':' | ',') { - cleaned.push('.'); - } - - // Word count of the *cleaned but not padded* text — padding is whitespace - // only and would just lie to the threshold check below. - let word_count = cleaned.split_whitespace().count(); - - let (final_text, max_frames) = if word_count <= SHORT_PROMPT_WORD_THRESHOLD { - let mut padded = String::with_capacity(cleaned.len() + SHORT_PROMPT_PAD_SPACES); - for _ in 0..SHORT_PROMPT_PAD_SPACES { - padded.push(' '); - } - padded.push_str(&cleaned); - (padded, Some(SHORT_PROMPT_MAX_FRAMES)) - } else { - // For everything ≥5 words, fall back to upstream defaults. Overriding - // these is what caused the "first 'yep' is static" regression — the - // upstream LM has been tuned for `frames_after_eos = 3` and - // `max_frames = 500`, and there's no clear win in second-guessing. - (cleaned, None) - }; - - Some(PreparedPrompt { - text: final_text, - max_frames, - }) -} - -/// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. -/// -/// Centralised so the regression test below can assert that we **never** -/// emit a `frames_after_eos` override — the previous attempt to override -/// that knob (setting it to 1 for ≥5-word inputs) clipped the leading -/// audio of multi-clause sentences (the "first 'yep' is static" bug on -/// 2026-05-18). The upstream sherpa-onnx default of 3 is what we want, and -/// the right way to keep it is to not set it at all. -fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { - prepared.max_frames.map(|mf| { - let mut h: HashMap = HashMap::with_capacity(1); - h.insert("max_frames".to_string(), serde_json::Value::from(mf)); - h - }) -} - -impl PocketTts { - /// Synthesise `text` with the given reference voice. - /// - /// `_lang` and `_steps` are accepted for API compatibility with the - /// previous Kokoro engine. Pocket TTS infers language from the input text - /// directly and is a one-step consistency model. Returns an empty buffer - /// for whitespace-only input. - pub fn synth_chunk( - &self, - text: &str, - _lang: &str, - style: &VoiceStyle, - _steps: usize, - ) -> Result, String> { - // Mirror upstream pocket-tts prompt prep — without this short or - // unpunctuated inputs can cause the LM's EOS logit to never trip, - // producing up to 40 s of "monster breathing" garbage on the first - // utterance. See `prepare_pocket_prompt` for the full recipe. - let prepared = match prepare_pocket_prompt(text) { - Some(p) => p, - None => return Ok(Vec::new()), - }; - - // Per-call generation hints sherpa-onnx forwards to - // `offline-tts-pocket-impl.h`. We only override `max_frames`, and - // only for short padded prompts where we have a tight expectation - // on output length — that bounds the original runaway without - // disturbing the rest of the LM sampling envelope. See - // `prepare_pocket_prompt` docs for the regression history. - let extra = build_generation_extra(&prepared); - - let cfg = GenerationConfig { - num_steps: SYNTH_NUM_STEPS, - silence_scale: SYNTH_SILENCE_SCALE, - reference_audio: Some(style.samples.clone()), - reference_sample_rate: style.sample_rate, - extra, - // `speed` stays at its default: the Pocket impl never reads it - // (see the engine-contract note in the module docs). - ..Default::default() - }; - - // No progress callback — synthesis is fast enough that returning the - // whole buffer at once keeps the lookahead pipelining in `tts.rs` - // simple. `None:: bool>` pins the callback type for the - // `generate_with_config` generic parameter. - let audio = self - .inner - .generate_with_config(&prepared.text, &cfg, None:: bool>) - .ok_or_else(|| { - format!( - "Pocket TTS synthesis failed for text ({} chars)", - prepared.text.len() - ) - })?; - - let sample_rate = audio.sample_rate(); - if sample_rate != SAMPLE_RATE as i32 { - eprintln!( - "buzz-desktop: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ - (expected {SAMPLE_RATE}Hz); playback speed may be wrong" - ); - } - - Ok(audio.samples().to_vec()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // ── prepare_pocket_prompt ──────────────────────────────────────────────── - - #[test] - fn prepare_prompt_returns_none_for_empty_input() { - assert!(prepare_pocket_prompt("").is_none()); - assert!(prepare_pocket_prompt(" ").is_none()); - assert!(prepare_pocket_prompt("\n\t ").is_none()); - } - - /// Helper: the exact leading sequence prepended to every short prompt — - /// 8 spaces of padding (upstream's cold-start mitigation). - /// Centralising this keeps the assertions readable. - fn short_prefix() -> String { - " ".repeat(SHORT_PROMPT_PAD_SPACES) - } - - #[test] - fn prepare_prompt_pads_and_capitalizes_one_word() { - // The "yep" case Tyler hit in production — bare lowercase one-word - // utterance with no punctuation. Must be padded with the short-prompt - // space pad, capitalized, terminated, with a tight `max_frames` cap - // to bound runaway gen. - let out = prepare_pocket_prompt("yep").expect("non-empty"); - assert_eq!(out.text, format!("{}Yep.", short_prefix())); - assert_eq!(out.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - const { - assert!( - SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT, - "short cap must be tighter than the upstream default" - ); - } - } - - #[test] - fn prepare_prompt_preserves_existing_punctuation() { - let out = prepare_pocket_prompt("yes!").expect("non-empty"); - assert_eq!(out.text, format!("{}Yes!", short_prefix())); // exclamation kept - let out = prepare_pocket_prompt("really?").expect("non-empty"); - assert_eq!(out.text, format!("{}Really?", short_prefix())); - } - - #[test] - fn prepare_prompt_threshold_is_inclusive_at_four_words() { - // 4 words = short (padded + tight max_frames); 5 words = long - // (no padding, no overrides — upstream defaults stand). - let four = prepare_pocket_prompt("one two three four").expect("non-empty"); - assert_eq!( - four.text, - format!("{}One two three four.", short_prefix()), - "four-word input should get exactly the space pad" - ); - assert_eq!(four.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - - let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); - assert!( - !five.text.starts_with(' '), - "five-word input should NOT be padded" - ); - assert_eq!( - five.max_frames, None, - "long inputs must leave sherpa-onnx's max_frames default in place" - ); - } - - #[test] - fn prepare_prompt_does_not_pad_long_text() { - let long = "This is a longer sentence that the model should handle just fine."; - let out = prepare_pocket_prompt(long).expect("non-empty"); - assert!(!out.text.starts_with(' ')); - assert_eq!(out.max_frames, None); - assert!(out.text.ends_with('.')); - } - - #[test] - fn prepare_prompt_collapses_whitespace() { - let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); - // 3 words → short → padded. Interior whitespace collapsed. - assert_eq!(out.text, format!("{}Hello world friend.", short_prefix())); - } - - #[test] - fn prepare_prompt_does_not_double_capitalize_already_uppercase() { - let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); - assert_eq!(out.text, format!("{}HELLO there.", short_prefix())); - } - - #[test] - fn prepare_prompt_handles_non_ascii_first_letter() { - // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce - // mojibake. - let out = prepare_pocket_prompt("дa").expect("non-empty"); - assert!(out.text.contains("Дa.")); - } - - /// REGRESSION GUARD: short prompts must receive *only* whitespace - /// padding — no sacrificial text. A previous revision prepended a - /// `". . "` cold-start absorber and trimmed the rendered audio back out - /// with an amplitude threshold that could eat soft word onsets. If - /// non-whitespace ever reappears in the pad, the synth output will - /// contain audio for text the user never wrote. - #[test] - fn prepare_prompt_pad_is_whitespace_only() { - let out = prepare_pocket_prompt("I'm happy.").expect("non-empty"); - let pad_len = out.text.len() - "I'm happy.".len(); - assert!( - out.text[..pad_len].chars().all(|c| c == ' '), - "short-prompt pad must be spaces only, got {:?}", - &out.text[..pad_len] - ); - assert_eq!(out.text, format!("{}I'm happy.", short_prefix())); - } - - // ── build_generation_extra ─────────────────────────────────────────────── - // - // These tests pin down a behaviour we've now regressed twice on: - // 1) Not padding/punctuating short inputs → 40 s of "monster breathing" - // (pre-773a2a1). - // 2) Setting `frames_after_eos = 1` on long inputs → clipped leading - // audio of multi-clause sentences, e.g. "Yep, I can hear you. …" - // came out as a static burst (the 773a2a1 regression Tyler hit on - // 2026-05-18 ~14:30 UTC). - // - // The contract we enforce going forward: we **only** override - // `max_frames`, and only for ≤4-word inputs. Every other knob is left - // at sherpa-onnx's documented default (notably `frames_after_eos = 3`). - - #[test] - fn build_extra_short_prompt_sets_only_max_frames() { - let prepared = prepare_pocket_prompt("yep").expect("non-empty"); - let extra = build_generation_extra(&prepared).expect("short prompts get extra"); - // Exactly one key — `max_frames` — and nothing else. - assert_eq!(extra.len(), 1, "extra has unexpected keys: {extra:?}"); - assert_eq!( - extra.get("max_frames"), - Some(&serde_json::Value::from(SHORT_PROMPT_MAX_FRAMES)) - ); - assert!( - !extra.contains_key("frames_after_eos"), - "frames_after_eos must never be set — upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT} is what we want" - ); - } - - #[test] - fn build_extra_long_prompt_is_none() { - // ≥5 words: no extras at all. This is the key fix for the "first - // 'yep' in 'Yep, I can hear you. …' is static" regression — we - // were previously forcing `frames_after_eos = 1` on this path. - let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); - assert_eq!( - build_generation_extra(&prepared), - None, - "long prompts must not override any LM knob" - ); - } - - #[test] - fn build_extra_never_lowers_frames_after_eos_for_any_word_count() { - // Sweep a range of prompt lengths and assert the `extra` map (when - // present) never carries a `frames_after_eos` override that's lower - // than the upstream sherpa-onnx default. Implemented as a structural - // check — we just never set the key — but worth a property test in - // case someone reintroduces the override in the future. - let prompts: &[&str] = &[ - "hi", - "hi there", - "yes please", - "one two three four", - "one two three four five", - "a slightly longer reply, hopefully fine", - "This is a multi-clause sentence. It has two parts.", - "really really really really really long prompt with lots of words just to be sure", - ]; - for &p in prompts { - let prepared = prepare_pocket_prompt(p).expect("non-empty"); - if let Some(extra) = build_generation_extra(&prepared) { - if let Some(v) = extra.get("frames_after_eos") { - let n = v.as_i64().expect("frames_after_eos should be int"); - assert!( - n >= SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT as i64, - "prompt {p:?} set frames_after_eos={n}, below upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT}" - ); - } - } - } - } - - #[test] - fn short_prompt_max_frames_is_below_upstream_default() { - // Sanity: the override only ever *lowers* the cap, never raises it. - const { - assert!(SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT); - } - // …and is still large enough for a one-to-four-word reply. At Mimi's - // 12.5 Hz frame rate, 100 frames = 8 s, which is roomy. - const { - assert!(SHORT_PROMPT_MAX_FRAMES >= 50, "would risk truncation"); - } - } -} +pub use buzz_voice_pkg::pocket::*; diff --git a/desktop/src-tauri/src/huddle/state.rs b/desktop/src-tauri/src/huddle/state.rs index 876c2d688b..55382a9880 100644 --- a/desktop/src-tauri/src/huddle/state.rs +++ b/desktop/src-tauri/src/huddle/state.rs @@ -202,8 +202,10 @@ impl HuddleState { /// to invalidate in-flight transcription tasks without losing the generation. pub(crate) fn reset_preserving_generation(&mut self) { let gen = Arc::clone(&self.session_generation); + let tts_enabled = self.tts_enabled; *self = Self::default(); self.session_generation = gen; + self.tts_enabled = tts_enabled; } } @@ -233,3 +235,20 @@ pub fn emit_huddle_state(app: &tauri::AppHandle, state: &HuddleState) { pub struct HuddleJoinInfo { pub ephemeral_channel_id: String, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn teardown_preserves_installation_global_tts_preference() { + let mut state = HuddleState { + tts_enabled: false, + phase: HuddlePhase::Active, + ..HuddleState::default() + }; + state.reset_preserving_generation(); + assert!(!state.tts_enabled); + assert_eq!(state.phase, HuddlePhase::Idle); + } +} diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 63a435cd8e..35b6328b8c 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -35,10 +35,11 @@ //! can gate microphone input while the agent is speaking. use std::{ + collections::VecDeque, num::NonZero, path::PathBuf, sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, mpsc::{self, SyncSender}, Arc, Mutex, MutexGuard, PoisonError, }, @@ -46,9 +47,13 @@ use std::{ time::Duration, }; -use super::pocket::{load_text_to_speech, load_voice_style, SAMPLE_RATE, VOICE_FILE_EXT}; +use super::pocket::{load_text_to_speech, load_voice_style, SynthesisOutcome, SAMPLE_RATE}; use super::preprocessing::{preprocess_for_tts, split_sentences}; +#[path = "tts_voice_transition.rs"] +mod voice_transition; +use voice_transition::*; + // ── Constants ───────────────────────────────────────────────────────────────── /// Maximum number of queued text items. @@ -122,7 +127,7 @@ const INTER_SENTENCE_SILENCE: f32 = 0.1; #[derive(Debug)] pub struct TtsPipeline { /// Send preprocessed text into the pipeline. - text_tx: SyncSender, + text_tx: SyncSender, /// `true` while the agent is speaking. Shared with the STT pipeline for gating. #[allow(dead_code)] pub tts_active: Arc, @@ -132,38 +137,25 @@ pub struct TtsPipeline { /// Kept alive here so the Arc isn't dropped — the worker holds a clone. #[allow(dead_code)] cancel: Arc, - /// Voice name (e.g. "reference_sample"). Stored for future voice-switching support. - #[allow(dead_code)] - voice: String, + /// Internal cancellation used only for voice changes. Kept separate so a + /// concurrent human barge-in always clears every queued message. + voice_cancel: Arc, + /// Selected manifest voice. The worker reloads only the lightweight style + /// when this changes; the warmed Pocket engine and audio player stay alive. + voice: Arc>, + /// Tags messages so a voice change drops only pre-change queue entries. + voice_generation: Arc, + /// Completed after the worker drains pre-change text and installs the new style. + voice_change_ack: VoiceChangeAck, /// Worker thread handle — taken on drop to join cleanly. thread: Option>, } impl TtsPipeline { - /// Spawn the TTS pipeline thread using the default voice. - /// - /// `model_dir` must contain the Pocket TTS files declared by `huddle::models` - /// (the five ONNX sessions, the two JSON tables, and `.wav`). + /// Spawn the TTS pipeline thread with a manifest-backed voice name. /// - /// `tts_active` is set to `true` while audio is playing and `false` when idle. - /// Pass the same `Arc` to the STT pipeline to gate microphone input. - /// - /// `cancel` is the shared barge-in flag from `HuddleState.tts_cancel`. Pass the - /// same `Arc` to the STT pipeline so both sides reference the same flag for the - /// entire huddle session — no stale references after pipeline restarts. - pub fn new( - model_dir: PathBuf, - tts_active: Arc, - cancel: Arc, - output_device: Option, - ) -> Result { - use super::pocket::DEFAULT_VOICE; - Self::new_with_voice(model_dir, tts_active, cancel, DEFAULT_VOICE, output_device) - } - - /// Spawn the TTS pipeline thread with a specific voice name. Today only the - /// bundled default voice (see `pocket::DEFAULT_VOICE`) is shipped; other - /// names will surface a clear error from `load_voice_style`. + /// `cancel` is shared with STT for barge-in. The same handle survives voice + /// changes so the warmed Pocket engine is retained. pub fn new_with_voice( model_dir: PathBuf, tts_active: Arc, @@ -171,14 +163,21 @@ impl TtsPipeline { voice: &str, output_device: Option, ) -> Result { - let (text_tx, text_rx) = mpsc::sync_channel::(TEXT_QUEUE_DEPTH); + let (text_tx, text_rx) = mpsc::sync_channel::(TEXT_QUEUE_DEPTH); let shutdown = Arc::new(AtomicBool::new(false)); // cancel is passed in from HuddleState.tts_cancel — shared with STT for barge-in. let shutdown_worker = Arc::clone(&shutdown); let cancel_worker = Arc::clone(&cancel); + let voice_cancel = Arc::new(AtomicBool::new(false)); + let worker_voice_cancel = Arc::clone(&voice_cancel); let tts_active_worker = Arc::clone(&tts_active); - let voice_name = voice.to_string(); + let voice = Arc::new(Mutex::new(voice.to_string())); + let voice_worker = Arc::clone(&voice); + let voice_generation = Arc::new(AtomicU64::new(1)); + let worker_voice_generation = Arc::clone(&voice_generation); + let voice_change_ack = Arc::new(Mutex::new(None)); + let worker_voice_change_ack = Arc::clone(&voice_change_ack); let model_dir_worker = model_dir.clone(); let handle = thread::Builder::new() @@ -186,11 +185,15 @@ impl TtsPipeline { .spawn(move || { tts_worker( model_dir_worker, - voice_name, + ( + voice_worker, + worker_voice_generation, + worker_voice_change_ack, + ), text_rx, tts_active_worker, shutdown_worker, - cancel_worker, + (cancel_worker, worker_voice_cancel), output_device, ) }) @@ -201,7 +204,10 @@ impl TtsPipeline { tts_active, shutdown, cancel, - voice: voice.to_string(), + voice_cancel, + voice, + voice_generation, + voice_change_ack, thread: Some(handle), }) } @@ -211,10 +217,49 @@ impl TtsPipeline { /// Non-blocking. Returns `Err` if the queue is full (bounded at /// `TEXT_QUEUE_DEPTH`) — caller may log and discard. pub fn speak(&self, text: String) -> Result<(), String> { - self.text_tx.try_send(text).map_err(|e| { - eprintln!("buzz-desktop: TTS queue saturated, dropping message: {e}"); - format!("TTS queue full, dropping: {e}") - }) + self.text_tx + .try_send(QueuedText { + generation: self.voice_generation.load(Ordering::Acquire), + text, + }) + .map_err(|e| { + eprintln!("buzz-desktop: TTS queue saturated, dropping message: {e}"); + format!("TTS queue full, dropping: {e}") + }) + } + + /// Clone the bounded queue sender so callers can apply backpressure without + /// holding the huddle mutex. Disabling TTS drops the receiver and unblocks + /// any waiting sender while the shared cancellation flag stops playback. + pub(crate) fn text_sender(&self) -> TtsTextSender { + TtsTextSender { + text_tx: self.text_tx.clone(), + generation: self.voice_generation.load(Ordering::Acquire), + } + } + + /// Select a bundled Pocket voice for subsequent speech. + /// + /// Current playback and queued text are cancelled immediately so content + /// cannot continue in the old voice. The worker keeps its warmed inference + /// engine and reloads only the reference style before the next utterance. + pub fn select_voice(&self, voice: &str) -> Option> { + begin_voice_change( + &self.voice, + &self.voice_generation, + &self.voice_cancel, + &self.voice_change_ack, + voice, + ) + } + + /// Reconcile the voice of a pipeline that has not been published yet. + /// + /// No caller can enqueue text before publication, so raising the shared + /// cancellation flag here would create a race that could discard the first + /// message queued immediately after installation. + pub(crate) fn select_voice_before_publish(&self, voice: &str) { + *self.voice.lock().unwrap_or_else(|error| error.into_inner()) = voice.to_string(); } /// Signal the worker thread to stop. @@ -244,13 +289,15 @@ impl Drop for TtsPipeline { fn tts_worker( model_dir: PathBuf, - voice_name: String, - text_rx: mpsc::Receiver, + voice_state: WorkerVoiceState, + text_rx: mpsc::Receiver, tts_active: Arc, shutdown: Arc, - cancel: Arc, + cancel_signals: WorkerCancelSignals, output_device: Option, ) { + let (selected_voice, voice_generation, voice_change_ack) = voice_state; + let (cancel, voice_cancel) = cancel_signals; // ── 1. Initialise TTS engine ────────────────────────────────────────────── let model_dir_str = model_dir.to_string_lossy().to_string(); @@ -261,20 +308,34 @@ fn tts_worker( "buzz-desktop: TTS engine init failed (model_dir={}): {e}. TTS disabled.", model_dir.display() ); - drain_until_shutdown(text_rx, &shutdown); + drain_tts_until_shutdown( + text_rx, + &shutdown, + (&cancel, &voice_cancel), + &voice_change_ack, + ); return; } }; // ── 2. Load voice style ─────────────────────────────────────────────────── - let voice_path = model_dir.join(format!("{voice_name}.{VOICE_FILE_EXT}")); - let style = match load_voice_style(&voice_path) { + let mut voice_name = selected_voice + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + let voice_path = voice_path(&model_dir, &voice_name); + let mut style = match load_voice_style(&voice_path) { Ok(s) => s, Err(e) => { eprintln!( "buzz-desktop: TTS voice style load failed ({voice_name}): {e}. TTS disabled." ); - drain_until_shutdown(text_rx, &shutdown); + drain_tts_until_shutdown( + text_rx, + &shutdown, + (&cancel, &voice_cancel), + &voice_change_ack, + ); return; } }; @@ -307,7 +368,12 @@ fn tts_worker( Ok(h) => h, Err(e) => { eprintln!("buzz-desktop: TTS audio output failed: {e}. TTS disabled."); - drain_until_shutdown(text_rx, &shutdown); + drain_tts_until_shutdown( + text_rx, + &shutdown, + (&cancel, &voice_cancel), + &voice_change_ack, + ); return; } }; @@ -377,6 +443,7 @@ fn tts_worker( let monitor = { let player = Arc::clone(&player); let cancel = Arc::clone(&cancel); + let voice_cancel = Arc::clone(&voice_cancel); let tts_active = Arc::clone(&tts_active); let stop = Arc::clone(&monitor_stop); let player_ops = Arc::clone(&player_ops); @@ -384,12 +451,12 @@ fn tts_worker( .name("tts-barge-in-monitor".into()) .spawn(move || { while !stop.load(Ordering::Acquire) { - if cancel.load(Ordering::Acquire) { + if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) { let _ops = lock_player_ops(&player_ops); // Re-check under the lock: the worker may have // consumed this cancel (and appended fresh audio) // between the load above and the lock acquisition. - if cancel.load(Ordering::Acquire) { + if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) { // clear() pauses the persistent player; play() // un-pauses (see handle_cancel_or_shutdown). // Idempotent — safe to repeat every tick until @@ -423,13 +490,16 @@ fn tts_worker( // idle branch below uses it to decide when to drop `tts_active` and to // arm a fresh lead-in cushion for the next utterance. let mut first_append = true; + let mut deferred_text = VecDeque::new(); loop { + let mut no_current_text = None; if handle_cancel_or_shutdown( - &cancel, + (&cancel, &voice_cancel), &shutdown, &tts_active, - &text_rx, + (&text_rx, &mut deferred_text, &mut no_current_text), + &voice_change_ack, Some((&player, &player_ops)), ) { if shutdown.load(Ordering::Acquire) { @@ -441,28 +511,42 @@ fn tts_worker( continue; } - let raw_text = match text_rx.recv_timeout(RECV_TIMEOUT) { - Ok(t) => t, - Err(mpsc::RecvTimeoutError::Timeout) => { - // Nothing queued. If playback has also finished, the agent - // has gone quiet — release the mic gate and reset the - // lead-in so the next utterance gets a fresh cushion. - if player.empty() && !first_append { - tts_active.store(false, Ordering::Release); - first_append = true; + // Voice changes cancel the old utterance/queue and are observed here, + // before receiving subsequent text. A bad bundled asset falls back to + // Mary without discarding the already-warmed Pocket engine. + let voice_ready = + reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style); + acknowledge_voice_change(&voice_change_ack, &voice_cancel); + if !voice_ready { + continue; + } + + let mut queued_text = Some(match deferred_text.pop_front() { + Some(text) => text, + None => match text_rx.recv_timeout(RECV_TIMEOUT) { + Ok(text) => text, + Err(mpsc::RecvTimeoutError::Timeout) => { + // Nothing queued. If playback has also finished, the agent + // has gone quiet — release the mic gate and reset the + // lead-in so the next utterance gets a fresh cushion. + if player.empty() && !first_append { + tts_active.store(false, Ordering::Release); + first_append = true; + } + continue; } - continue; - } - Err(mpsc::RecvTimeoutError::Disconnected) => break, - }; + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }, + }); // Check cancel again after unblocking — a cancel may have arrived // while we were waiting. if handle_cancel_or_shutdown( - &cancel, + (&cancel, &voice_cancel), &shutdown, &tts_active, - &text_rx, + (&text_rx, &mut deferred_text, &mut queued_text), + &voice_change_ack, Some((&player, &player_ops)), ) { if shutdown.load(Ordering::Acquire) { @@ -471,6 +555,21 @@ fn tts_worker( first_append = true; continue; } + let Some(queued_text) = queued_text else { + continue; + }; + if queued_text.generation < voice_generation.load(Ordering::Acquire) { + continue; + } + let raw_text = queued_text.text; + + // The selected voice can change while this worker is blocked in + // recv_timeout. Reconcile again after receipt so the first message + // queued after an unpublished pipeline is installed cannot use the + // voice captured when construction began. + if !reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style) { + continue; + } // If playback already drained while we were waiting for this item, // the agent is silent — release the mic gate BEFORE preprocessing/ @@ -505,11 +604,13 @@ fn tts_worker( let chunks = group_sentences_into_chunks(&sentences, MAX_CHUNK_CHARS); for chunk in &chunks { + let mut no_current_text = None; if handle_cancel_or_shutdown( - &cancel, + (&cancel, &voice_cancel), &shutdown, &tts_active, - &text_rx, + (&text_rx, &mut deferred_text, &mut no_current_text), + &voice_change_ack, Some((&player, &player_ops)), ) { first_append = true; @@ -521,8 +622,15 @@ fn tts_worker( continue; } - match engine.synth_chunk(text, "en", &style, SYNTH_STEPS) { - Ok(samples) if !samples.is_empty() => { + let synth_cancel = Arc::clone(&cancel); + let synth_voice_cancel = Arc::clone(&voice_cancel); + let synth_shutdown = Arc::clone(&shutdown); + match engine.synth_chunk_interruptible(text, "en", &style, SYNTH_STEPS, move || { + synth_cancel.load(Ordering::Acquire) + || synth_voice_cancel.load(Ordering::Acquire) + || synth_shutdown.load(Ordering::Acquire) + }) { + Ok(SynthesisOutcome::Complete(samples)) if !samples.is_empty() => { let mut audio = clamp_to_full_scale(samples); // Fade-out only — fading-in would attenuate the consonant // onset (see `apply_fade_out` docstring + the @@ -549,7 +657,7 @@ fn tts_worker( // does the full consume (drain queue, reset lead-in) on // the next iteration. let _ops = lock_player_ops(&player_ops); - if cancel.load(Ordering::Acquire) { + if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) { // Nothing appended; the loop-top consume re-arms // `first_append` (the flag is still set — the worker // is its only consumer). @@ -563,7 +671,11 @@ fn tts_worker( // See crossfire review C3. tts_active.store(true, Ordering::Release); } - Ok(_) => {} + Ok(SynthesisOutcome::Complete(_)) => {} + Ok(SynthesisOutcome::Interrupted) => { + first_append = true; + break; + } Err(e) => { eprintln!("buzz-desktop: TTS synth failed: {e}"); } @@ -582,11 +694,35 @@ fn tts_worker( let _ = handle.join(); } + finish_voice_change_ack(&voice_change_ack); tts_active.store(false, Ordering::Release); } // ── Helpers ─────────────────────────────────────────────────────────────────── +fn drain_tts_until_shutdown( + text_rx: mpsc::Receiver, + shutdown: &AtomicBool, + cancel_signals: CancelSignals<'_>, + voice_change_ack: &VoiceChangeAck, +) { + let (cancel, voice_cancel) = cancel_signals; + loop { + if cancel.swap(false, Ordering::AcqRel) | voice_cancel.swap(false, Ordering::AcqRel) { + while text_rx.try_recv().is_ok() {} + } + acknowledge_voice_change(voice_change_ack, voice_cancel); + if shutdown.load(Ordering::Acquire) { + break; + } + match text_rx.recv_timeout(RECV_TIMEOUT) { + Ok(_) | Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => break, + } + } + finish_voice_change_ack(voice_change_ack); +} + /// Check for cancel or shutdown. Returns `true` if the caller should break/continue. /// On cancel: drains the text queue and clears the cancel flag. /// @@ -595,12 +731,15 @@ fn tts_worker( /// it is serialized with the monitor's stale-branch re-check (see the monitor /// block in `tts_worker`). fn handle_cancel_or_shutdown( - cancel: &AtomicBool, + cancel_signals: CancelSignals<'_>, shutdown: &AtomicBool, tts_active: &AtomicBool, - text_rx: &mpsc::Receiver, + text_state: CancelTextState<'_>, + voice_change_ack: &VoiceChangeAck, player: Option<(&rodio::Player, &Mutex<()>)>, ) -> bool { + let (cancel, voice_cancel) = cancel_signals; + let (text_rx, deferred_text, current_text) = text_state; if shutdown.load(Ordering::Acquire) { if let Some((p, ops)) = player { let _ops = lock_player_ops(ops); @@ -609,7 +748,24 @@ fn handle_cancel_or_shutdown( tts_active.store(false, Ordering::Release); return true; } - if cancel.load(Ordering::Acquire) { + if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) { + // Serialize with begin_voice_change so the generation boundary and + // cancel consumption are observed as one transition. + let pending_voice_change = voice_change_ack + .lock() + .unwrap_or_else(|error| error.into_inner()); + // Consume at the serialization point. A later barge-in remains true + // for the next pass instead of being overwritten after queue cleanup. + let barge_in = cancel.swap(false, Ordering::AcqRel); + voice_cancel.store(false, Ordering::Release); + let preserve_generation = (!barge_in) + .then(|| { + pending_voice_change + .as_ref() + .map(|pending| pending.generation) + }) + .flatten(); + retain_cancelled_text(deferred_text, current_text, text_rx, preserve_generation); if let Some((p, ops)) = player { let _ops = lock_player_ops(ops); // `Player::clear()` removes queued sources AND pauses the player @@ -622,11 +778,6 @@ fn handle_cancel_or_shutdown( // Consume the flag under the lock: once released with // `cancel == false`, the monitor's stale branch no-ops instead // of clearing the fresh post-cancel utterance. - while text_rx.try_recv().is_ok() {} - cancel.store(false, Ordering::Release); - } else { - while text_rx.try_recv().is_ok() {} - cancel.store(false, Ordering::Release); } tts_active.store(false, Ordering::Release); return true; @@ -768,11 +919,11 @@ fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec, + tokio::sync::oneshot::Receiver<()>, +); + +const VOICE_AVAILABILITY_BUNDLED: &str = "bundled"; +const VOICE_AVAILABILITY_INSTALLED: &str = "installed"; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct VoiceRegistryEntry { + /// Stable identity, never derived from or merged by the display name. + /// + /// Built-ins use `backend:slug`. Future imports use + /// `pocket:imported:` so two clips with the same + /// editable label remain distinct. + pub key: String, + pub display_name: String, + pub backend: String, + pub backend_name: String, + /// Client-local state: bundled, installed, downloadable, or unavailable. + pub availability: String, + pub fallback_key: Option, + pub reference_file: Option, + pub provenance: VoiceProvenance, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct VoiceProvenance { + pub source: String, + pub content_hash: Option, + pub license: Option, + pub source_url: Option, +} + +/// Ordered, backend-qualified preferences shared by global and agent settings. +/// +/// Unknown but well-formed keys remain persisted because a different client +/// may have that backend installed. Resolution is always local. +pub type VoicePreferences = Vec; + +/// Bundled Pocket voices available without local imports. +pub fn bundled_voice_registry() -> Vec { + vec![ + VoiceRegistryEntry { + key: MARY_VOICE_KEY.to_string(), + display_name: "Mary".to_string(), + backend: POCKET_BACKEND_ID.to_string(), + backend_name: "Pocket TTS".to_string(), + availability: VOICE_AVAILABILITY_BUNDLED.to_string(), + fallback_key: None, + reference_file: Some("reference_sample.wav".to_string()), + provenance: VoiceProvenance { + source: "bundled".to_string(), + content_hash: Some( + "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f" + .to_string(), + ), + license: Some("CC-BY-4.0".to_string()), + source_url: Some("https://datashare.ed.ac.uk/handle/10283/3443".to_string()), + }, + }, + VoiceRegistryEntry { + key: MARIUS_VOICE_KEY.to_string(), + display_name: "Marius".to_string(), + backend: POCKET_BACKEND_ID.to_string(), + backend_name: "Pocket TTS".to_string(), + availability: VOICE_AVAILABILITY_BUNDLED.to_string(), + fallback_key: Some(MARY_VOICE_KEY.to_string()), + reference_file: Some("marius.wav".to_string()), + provenance: VoiceProvenance { + source: "bundled".to_string(), + content_hash: Some( + "076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16" + .to_string(), + ), + license: Some("CC0-1.0".to_string()), + source_url: Some( + "https://huggingface.co/kyutai/tts-voices/blob/323332d33f997de8394f24a193e1a76df720e01a/voice-donations/Selfie.wav" + .to_string(), + ), + }, + }, + ] +} + +/// Cross-backend registry of bundled and locally installed voices. +pub fn voice_registry(app: &AppHandle) -> Vec { + let mut registry = bundled_voice_registry(); + match super::tts_voice_import::load_registry(app) { + Ok(imported) => registry.extend(imported.into_iter().map(|voice| VoiceRegistryEntry { + key: voice.key, + display_name: voice.display_name, + backend: POCKET_BACKEND_ID.to_string(), + backend_name: "Pocket TTS".to_string(), + availability: VOICE_AVAILABILITY_INSTALLED.to_string(), + fallback_key: Some(MARY_VOICE_KEY.to_string()), + reference_file: Some(voice.file_name), + provenance: VoiceProvenance { + source: "local import".to_string(), + content_hash: Some(voice.content_hash), + license: None, + source_url: None, + }, + })), + Err(error) => { + eprintln!( + "buzz-desktop: {error}; imported Pocket voices are unavailable for this session" + ); + } + } + registry +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct TtsSettings { + pub version: u32, + pub agent_text_to_speech: bool, + pub voice_preferences: VoicePreferences, +} + +impl Default for TtsSettings { + fn default() -> Self { + Self { + version: CURRENT_VERSION, + agent_text_to_speech: true, + voice_preferences: vec![MARY_VOICE_KEY.to_string()], + } + } +} + +pub fn voice_by_key(app: &AppHandle, key: &str) -> Option { + voice_registry(app) + .into_iter() + .find(|voice| voice.key == key) +} + +fn is_qualified_voice_key(key: &str) -> bool { + key.split_once(':') + .is_some_and(|(backend, voice)| !backend.is_empty() && !voice.is_empty()) +} + +fn is_locally_available(availability: &str) -> bool { + matches!( + availability, + VOICE_AVAILABILITY_BUNDLED | VOICE_AVAILABILITY_INSTALLED + ) +} + +#[cfg(test)] +pub fn resolve_voice_for_backend( + preferences: &[String], + backend: &str, +) -> Result { + resolve_voice_for_backend_in_registry(preferences, backend, &bundled_voice_registry()) +} + +fn resolve_voice_for_backend_in_registry( + preferences: &[String], + backend: &str, + registry: &[VoiceRegistryEntry], +) -> Result { + preferences + .iter() + .filter_map(|key| registry.iter().find(|voice| voice.key == *key)) + .find(|voice| voice.backend == backend && is_locally_available(voice.availability.as_str())) + .or_else(|| { + registry.iter().find(|voice| { + voice.backend == backend + && voice.fallback_key.is_none() + && is_locally_available(voice.availability.as_str()) + }) + }) + .cloned() + .ok_or_else(|| format!("No locally available fallback voice for backend {backend}")) +} + +pub fn bundled_pocket_voice_reference(preferences: &[String]) -> String { + resolve_voice_for_backend_in_registry(preferences, POCKET_BACKEND_ID, &bundled_voice_registry()) + .ok() + .and_then(|voice| voice.reference_file) + .and_then(|file| file.strip_suffix(".wav").map(str::to_string)) + .unwrap_or_else(|| DEFAULT_VOICE.to_string()) +} + +pub fn pocket_voice_reference(app: &AppHandle, preferences: &[String]) -> Result { + let registry = voice_registry(app); + let voice = resolve_voice_for_backend_in_registry(preferences, POCKET_BACKEND_ID, ®istry)?; + if voice.key.starts_with("pocket:imported:") { + let imported = super::tts_voice_import::load_registry(app)? + .into_iter() + .find(|candidate| candidate.key == voice.key) + .ok_or_else(|| format!("Imported voice {} is unavailable", voice.display_name))?; + return super::tts_voice_import::resolve_file(app, &imported) + .map(|path| path.to_string_lossy().into_owned()); + } + Ok(voice + .reference_file + .and_then(|file| file.strip_suffix(".wav").map(str::to_string)) + .unwrap_or_else(|| DEFAULT_VOICE.to_string())) +} + +pub(crate) fn settings_path(app: &AppHandle) -> Result { + app.path() + .app_data_dir() + .map(|dir| dir.join(SETTINGS_FILE)) + .map_err(|error| format!("could not locate Buzz settings storage: {error}")) +} + +pub(crate) fn load_from_path(path: &Path) -> Result { + if !path.exists() { + return Ok(TtsSettings::default()); + } + let bytes = std::fs::read(path) + .map_err(|error| format!("could not read text-to-speech settings: {error}"))?; + let value: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|error| format!("text-to-speech settings are not valid JSON: {error}"))?; + + // Unversioned settings are incompatible with the V1 schema. Use + // deterministic V1 defaults rather than interpreting ambiguous fields. + if value.get("version").is_none() { + return Ok(TtsSettings::default()); + } + + let version = value + .get("version") + .and_then(serde_json::Value::as_u64) + .ok_or("text-to-speech settings version is invalid")?; + if version > u64::from(CURRENT_VERSION) { + return Err(format!( + "text-to-speech settings version {version} is newer than this Buzz build supports" + )); + } + + // Legacy V1 settings may contain one bare Pocket `voiceId`. Preserve the + // toggle and qualify it into the ordered cross-backend preference schema. + if value.get("voicePreferences").is_none() { + let legacy_voice = value + .get("voiceId") + .or_else(|| value.get("voice_id")) + .and_then(serde_json::Value::as_str) + .unwrap_or("mary"); + let voice_key = if is_qualified_voice_key(legacy_voice) { + legacy_voice.to_string() + } else { + format!("{POCKET_BACKEND_ID}:{legacy_voice}") + }; + return Ok(TtsSettings { + version: CURRENT_VERSION, + agent_text_to_speech: value + .get("agentTextToSpeech") + .and_then(serde_json::Value::as_bool) + .unwrap_or(true), + voice_preferences: vec![voice_key], + }); + } + + let mut settings: TtsSettings = serde_json::from_value(value) + .map_err(|error| format!("text-to-speech settings are invalid: {error}"))?; + settings.version = CURRENT_VERSION; + if settings.voice_preferences.is_empty() + || settings + .voice_preferences + .iter() + .any(|key| !is_qualified_voice_key(key)) + { + settings.voice_preferences = TtsSettings::default().voice_preferences; + } + Ok(settings) +} + +pub(crate) fn save_to_path(path: &Path, settings: &TtsSettings) -> Result<(), String> { + if settings.voice_preferences.is_empty() { + return Err("At least one voice preference is required".to_string()); + } + if let Some(key) = settings + .voice_preferences + .iter() + .find(|key| !is_qualified_voice_key(key)) + { + return Err(format!( + "Voice preference keys must be backend-qualified: {key}" + )); + } + let payload = serde_json::to_vec_pretty(settings) + .map_err(|error| format!("could not encode text-to-speech settings: {error}"))?; + atomic_write_json_restricted(path, &payload) + .map_err(|error| format!("could not save text-to-speech settings: {error}")) +} + +pub fn load_for_app(app: &AppHandle) -> (TtsSettings, Option) { + let result = settings_path(app).and_then(|path| load_from_path(&path)); + match result { + Ok(settings) => (settings, None), + Err(error) => { + eprintln!("buzz-desktop: {error}; preserving the file and using Mary for this session"); + (TtsSettings::default(), Some(error)) + } + } +} + +#[tauri::command] +pub fn get_tts_settings(state: State<'_, AppState>) -> Result { + if let Some(error) = state + .tts_settings_load_error + .lock() + .map_err(|lock_error| format!("text-to-speech settings lock poisoned: {lock_error}"))? + .clone() + { + return Err(format!( + "Voice settings could not be loaded and were left unchanged: {error}" + )); + } + state + .tts_settings + .lock() + .map(|settings| settings.clone()) + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}")) +} + +#[tauri::command] +pub fn list_voice_registry(app: AppHandle) -> Vec { + voice_registry(&app) +} + +fn ensure_settings_writable(state: &AppState) -> Result<(), String> { + if let Some(error) = state + .tts_settings_load_error + .lock() + .map_err(|lock_error| format!("text-to-speech settings lock poisoned: {lock_error}"))? + .as_ref() + { + return Err(format!( + "Voice settings were not saved because the existing file could not be loaded: {error}" + )); + } + Ok(()) +} + +fn cancel_huddle_speech( + huddle: &mut super::HuddleState, +) -> Option> { + huddle.tts_enabled = false; + huddle + .tts_cancel + .store(true, std::sync::atomic::Ordering::Release); + huddle.tts_pipeline.take() +} + +fn disable_tts_runtime(state: &AppState) -> Result<(), String> { + let old_pipeline = { + let mut huddle = state.huddle()?; + cancel_huddle_speech(&mut huddle) + }; + if let Some(ref pipeline) = old_pipeline { + pipeline.shutdown(); + } + drop(old_pipeline); + state.emit_huddle_state_changed(); + Ok(()) +} + +fn commit_effective_off(state: &AppState) -> Result<(), String> { + state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? + .agent_text_to_speech = false; + Ok(()) +} + +fn enable_tts_runtime(huddle: &mut HuddleState, voice: &str) -> Option { + huddle.tts_enabled = true; + // OFF removes the pipeline. Clear a prior cancellation only when enabling + // a fresh pipeline; an idempotent ON write must not erase a voice + // transition that the existing worker still needs to drain. + if huddle.tts_pipeline.is_none() { + huddle + .tts_cancel + .store(false, std::sync::atomic::Ordering::Release); + } + huddle.tts_pipeline.as_ref().and_then(|pipeline| { + pipeline + .select_voice(voice) + .map(|acknowledged| (Arc::clone(pipeline), acknowledged)) + }) +} + +async fn apply_tts_settings( + settings: TtsSettings, + app: &AppHandle, + state: &AppState, +) -> Result, String> { + if settings.version != CURRENT_VERSION { + return Err(format!( + "Unsupported text-to-speech settings version: {}", + settings.version + )); + } + + // OFF is safety-sensitive: stop current and queued speech before any disk + // I/O, and never resume it merely because persistence fails. + if !settings.agent_text_to_speech { + disable_tts_runtime(state)?; + commit_effective_off(state)?; + } + + ensure_settings_writable(state)?; + save_to_path(&settings_path(app)?, &settings)?; + + *state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? = + settings.clone(); + + let mut voice_change_wait = None; + if settings.agent_text_to_speech { + let (active, voice_change_ack) = { + let mut huddle = state.huddle()?; + let voice_reference = pocket_voice_reference(app, &settings.voice_preferences)?; + let voice_change_ack = enable_tts_runtime(&mut huddle, &voice_reference); + ( + matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active), + voice_change_ack, + ) + }; + voice_change_wait = voice_change_ack; + if active { + if let Err(error) = super::pipeline::maybe_start_tts_pipeline(state).await { + eprintln!("buzz-desktop: could not hot-start text to speech: {error}"); + } + } + state.emit_huddle_state_changed(); + } + Ok(voice_change_wait) +} + +fn current_settings(state: &AppState) -> Result { + state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}")) + .map(|settings| settings.clone()) +} + +async fn finish_voice_change(voice_change: Option) -> Result<(), String> { + let Some((pipeline, acknowledged)) = voice_change else { + return Ok(()); + }; + wait_for_voice_change_ack(acknowledged, VOICE_CHANGE_ACK_TIMEOUT, || { + pipeline.is_finished() + }) + .await +} + +async fn wait_for_voice_change_ack( + mut acknowledged: tokio::sync::oneshot::Receiver<()>, + timeout: Duration, + mut worker_is_finished: impl FnMut() -> bool, +) -> Result<(), String> { + let deadline = tokio::time::sleep(timeout); + tokio::pin!(deadline); + loop { + tokio::select! { + _ = &mut acknowledged => return Ok(()), + _ = &mut deadline => { + return Err( + "Pocket TTS is still finishing the previous voice. Turn Agent text to speech off and try again." + .to_string(), + ); + } + _ = tokio::time::sleep(Duration::from_millis(25)) => { + if worker_is_finished() { + return Ok(()); + } + } + } + } +} + +/// Compatibility command for the huddle speaker button. It updates the same +/// installation-global preference as Settings; there is no per-huddle override. +#[tauri::command] +pub async fn set_tts_enabled( + enabled: bool, + app: AppHandle, + state: State<'_, AppState>, +) -> Result { + let transition = state.tts_settings_transition.lock().await; + let mut settings = state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? + .clone(); + settings.agent_text_to_speech = enabled; + let voice_change = apply_tts_settings(settings, &app, &state).await?; + drop(transition); + finish_voice_change(voice_change).await?; + current_settings(&state) +} + +fn settings_with_pocket_voice( + settings: TtsSettings, + voice_key: &str, + app: &AppHandle, +) -> Result { + settings_with_pocket_voice_from_registry(settings, voice_key, &voice_registry(app)) +} + +fn settings_with_pocket_voice_from_registry( + mut settings: TtsSettings, + voice_key: &str, + registry: &[VoiceRegistryEntry], +) -> Result { + let voice = registry + .iter() + .find(|voice| voice.key == voice_key) + .ok_or_else(|| format!("Unknown voice: {voice_key}"))?; + if voice.backend != POCKET_BACKEND_ID || !is_locally_available(&voice.availability) { + return Err("The selected Pocket voice is not available on this device".to_string()); + } + let first_pocket_index = settings + .voice_preferences + .iter() + .position(|key| key.starts_with("pocket:")); + settings + .voice_preferences + .retain(|key| !key.starts_with("pocket:")); + let insert_at = first_pocket_index + .unwrap_or(settings.voice_preferences.len()) + .min(settings.voice_preferences.len()); + settings + .voice_preferences + .insert(insert_at, voice_key.to_string()); + Ok(settings) +} + +#[tauri::command] +pub async fn set_pocket_voice( + voice_key: String, + app: AppHandle, + state: State<'_, AppState>, +) -> Result { + let transition = state.tts_settings_transition.lock().await; + let settings = state + .tts_settings + .lock() + .map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? + .clone(); + let settings = settings_with_pocket_voice(settings, &voice_key, &app)?; + let voice_change = apply_tts_settings(settings, &app, &state).await?; + drop(transition); + finish_voice_change(voice_change).await?; + current_settings(&state) +} + +#[tauri::command] +pub async fn preview_pocket_voice( + voice_key: String, + app: AppHandle, + state: State<'_, AppState>, +) -> Result<(), String> { + let voice = + voice_by_key(&app, &voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?; + if voice.backend != POCKET_BACKEND_ID { + return Err("Only Pocket voices can be previewed in this build".to_string()); + } + if !models::is_tts_ready() { + return Err("Voice files are still downloading. Try preview again shortly.".to_string()); + } + let model_dir = models::tts_model_dir().ok_or("Pocket voice files are unavailable")?; + let output_device = state + .audio_output_device + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + let voice_name = pocket_voice_reference(&app, std::slice::from_ref(&voice_key))?; + tokio::task::spawn_blocking(move || { + let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let pipeline = super::tts::TtsPipeline::new_with_voice( + model_dir, + active.clone(), + cancel, + &voice_name, + output_device, + )?; + pipeline.speak("Hello! This is how I’ll read agent responses.".to_string())?; + let started = std::time::Instant::now(); + let mut heard_audio = false; + while started.elapsed() < std::time::Duration::from_secs(30) { + let is_active = active.load(std::sync::atomic::Ordering::Acquire); + heard_audio |= is_active; + if heard_audio && !is_active { + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + Err("Voice preview timed out. Check your audio output and try again.".to_string()) + }) + .await + .map_err(|error| format!("Voice preview task failed: {error}"))? +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TtsVoiceMutation { + pub settings: TtsSettings, + pub registry: Vec, +} + +#[tauri::command] +pub async fn import_pocket_voice( + app: AppHandle, + state: State<'_, AppState>, +) -> Result, String> { + let Some(imported) = super::tts_voice_import::pick_and_import(&app).await? else { + return Ok(None); + }; + let transition = state.tts_settings_transition.lock().await; + let settings = current_settings(&state)?; + let settings = settings_with_pocket_voice(settings, &imported.key, &app)?; + let voice_change = apply_tts_settings(settings, &app, &state).await?; + drop(transition); + finish_voice_change(voice_change).await?; + Ok(Some(TtsVoiceMutation { + settings: current_settings(&state)?, + registry: voice_registry(&app), + })) +} + +#[tauri::command] +pub async fn delete_pocket_voice( + voice_key: String, + app: AppHandle, + state: State<'_, AppState>, +) -> Result { + if !voice_key.starts_with("pocket:imported:") { + return Err("Bundled voices cannot be deleted".to_string()); + } + if voice_by_key(&app, &voice_key).is_none() { + return Err(format!("Unknown imported voice: {voice_key}")); + } + + let transition = state.tts_settings_transition.lock().await; + let current = current_settings(&state)?; + let selected = resolve_voice_for_backend_in_registry( + ¤t.voice_preferences, + POCKET_BACKEND_ID, + &voice_registry(&app), + ) + .is_ok_and(|voice| voice.key == voice_key); + let voice_change = if selected { + let fallback = settings_with_pocket_voice(current, MARY_VOICE_KEY, &app)?; + apply_tts_settings(fallback, &app, &state).await? + } else { + None + }; + drop(transition); + finish_voice_change(voice_change).await?; + super::tts_voice_import::delete(&app, &voice_key)?; + Ok(TtsVoiceMutation { + settings: current_settings(&state)?, + registry: voice_registry(&app), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn stalled_voice_change_returns_an_actionable_error() { + let (_keep_pending, acknowledged) = tokio::sync::oneshot::channel(); + + let error = wait_for_voice_change_ack(acknowledged, Duration::from_millis(1), || false) + .await + .expect_err("stalled worker should time out"); + + assert!(error.contains("Turn Agent text to speech off")); + } + + #[test] + fn idempotent_enable_preserves_an_existing_pipeline_cancel() { + let state = crate::app_state::build_app_state(); + let model_dir = tempfile::tempdir().expect("temp model dir"); + let cancel = state.huddle().expect("huddle state").tts_cancel.clone(); + let pipeline = Arc::new( + super::super::tts::TtsPipeline::new_with_voice( + model_dir.path().to_path_buf(), + Arc::new(std::sync::atomic::AtomicBool::new(false)), + Arc::clone(&cancel), + "reference_sample", + None, + ) + .expect("pipeline"), + ); + pipeline.shutdown(); + for _ in 0..100 { + if pipeline.is_finished() { + break; + } + std::thread::sleep(Duration::from_millis(1)); + } + assert!(pipeline.is_finished(), "test pipeline should stop"); + + cancel.store(true, std::sync::atomic::Ordering::Release); + let mut huddle = state.huddle().expect("huddle state"); + huddle.tts_pipeline = Some(pipeline); + + assert!(enable_tts_runtime(&mut huddle, "reference_sample").is_none()); + assert!(cancel.load(std::sync::atomic::Ordering::Acquire)); + } + + #[test] + fn defaults_are_backwards_compatible_and_use_mary() { + assert_eq!( + TtsSettings::default(), + TtsSettings { + version: 1, + agent_text_to_speech: true, + voice_preferences: vec!["pocket:mary".to_string()], + } + ); + } + + #[test] + fn v1_registry_has_exactly_the_two_reviewed_bundled_voices() { + assert_eq!( + bundled_voice_registry() + .iter() + .map(|voice| { + ( + voice.key.as_str(), + voice.display_name.as_str(), + voice.reference_file.as_deref(), + ) + }) + .collect::>(), + vec![ + ("pocket:mary", "Mary", Some("reference_sample.wav")), + ("pocket:marius", "Marius", Some("marius.wav")), + ] + ); + } + + #[test] + fn local_backend_resolution_uses_first_compatible_preference() { + let preferences = vec![ + "siri:aaron".to_string(), + MARIUS_VOICE_KEY.to_string(), + MARY_VOICE_KEY.to_string(), + "kokoro:af_heart".to_string(), + ]; + assert_eq!( + resolve_voice_for_backend(&preferences, POCKET_BACKEND_ID) + .expect("Pocket fallback") + .key, + MARIUS_VOICE_KEY + ); + } + + #[test] + fn unsupported_or_missing_preferences_fall_back_to_backend_default() { + let preferences = vec![ + "siri:aaron".to_string(), + "pocket:imported:deadbeef".to_string(), + ]; + assert_eq!( + resolve_voice_for_backend(&preferences, POCKET_BACKEND_ID) + .expect("Pocket fallback") + .key, + MARY_VOICE_KEY + ); + } + + #[test] + fn identity_is_qualified_key_not_display_label() { + assert!(is_qualified_voice_key("pocket:imported:audio-content-hash")); + assert_ne!(MARY_VOICE_KEY, MARIUS_VOICE_KEY); + let mut registry = bundled_voice_registry(); + registry[0].display_name = "Jim".to_string(); + registry[1].display_name = "Jim".to_string(); + assert_eq!(registry[0].display_name, registry[1].display_name); + assert_ne!(registry[0].key, registry[1].key); + assert_eq!( + registry + .iter() + .map(|voice| voice.key.as_str()) + .collect::>() + .len(), + registry.len() + ); + } + + #[test] + fn bundled_marius_asset_has_reviewed_size_and_wav_container() { + let bytes = include_bytes!("../../resources/pocket-voices/marius.wav"); + assert_eq!(bytes.len(), 480_044); + assert_eq!(&bytes[0..4], b"RIFF"); + assert_eq!(&bytes[8..12], b"WAVE"); + assert_eq!( + hex::encode(::digest(bytes)), + "076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16" + ); + } + + #[test] + fn migrates_unversioned_experiment_settings_to_v1_defaults() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(SETTINGS_FILE); + std::fs::write(&path, r#"{"voice":"legacy-experiment"}"#).expect("fixture write"); + assert_eq!( + load_from_path(&path).expect("migration"), + TtsSettings::default() + ); + } + + #[test] + fn migrates_bare_pocket_voice_id_to_qualified_preferences() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(SETTINGS_FILE); + std::fs::write( + &path, + r#"{"version":1,"agentTextToSpeech":false,"voiceId":"marius"}"#, + ) + .expect("fixture write"); + assert_eq!( + load_from_path(&path).expect("migration"), + TtsSettings { + version: 1, + agent_text_to_speech: false, + voice_preferences: vec![MARIUS_VOICE_KEY.to_string()], + } + ); + } + + #[test] + fn unknown_qualified_preferences_are_preserved_for_other_clients() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(SETTINGS_FILE); + std::fs::write( + &path, + r#"{"version":1,"agentTextToSpeech":false,"voicePreferences":["siri:aaron","pocket:imported:abc123"]}"#, + ) + .expect("fixture write"); + let settings = load_from_path(&path).expect("load"); + assert!(!settings.agent_text_to_speech); + assert_eq!( + settings.voice_preferences, + vec!["siri:aaron", "pocket:imported:abc123"] + ); + } + + #[test] + fn rejects_future_schema_versions_clearly() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join(SETTINGS_FILE); + std::fs::write( + &path, + r#"{"version":99,"agentTextToSpeech":true,"voicePreferences":["pocket:mary"]}"#, + ) + .expect("fixture write"); + assert!(load_from_path(&path) + .expect_err("future version should fail") + .contains("newer than this Buzz build supports")); + } + + #[test] + fn disabling_cancels_runtime_before_persistence_can_fail() { + let mut huddle = super::super::HuddleState { + tts_enabled: true, + ..super::super::HuddleState::default() + }; + assert!(!huddle.tts_cancel.load(std::sync::atomic::Ordering::Acquire)); + assert!(cancel_huddle_speech(&mut huddle).is_none()); + assert!(!huddle.tts_enabled); + assert!(huddle.tts_cancel.load(std::sync::atomic::Ordering::Acquire)); + } + + #[test] + fn pocket_voice_update_preserves_the_latest_toggle_and_other_backends() { + let current = TtsSettings { + agent_text_to_speech: false, + voice_preferences: vec!["siri:aaron".to_string(), MARY_VOICE_KEY.to_string()], + ..TtsSettings::default() + }; + let updated = settings_with_pocket_voice_from_registry( + current, + MARIUS_VOICE_KEY, + &bundled_voice_registry(), + ) + .expect("available voice"); + assert!(!updated.agent_text_to_speech); + assert_eq!( + updated.voice_preferences, + vec!["siri:aaron", MARIUS_VOICE_KEY] + ); + } + + #[test] + fn failed_off_persistence_cannot_be_undone_by_a_later_voice_update() { + let state = crate::app_state::build_app_state(); + commit_effective_off(&state).expect("commit effective OFF state"); + + // This models the next command after the OFF save fails: it must merge + // from effective memory state, not the stale last-persisted ON value. + let current = state.tts_settings.lock().expect("settings").clone(); + let voice_update = settings_with_pocket_voice_from_registry( + current, + MARIUS_VOICE_KEY, + &bundled_voice_registry(), + ) + .expect("available voice"); + assert!(!voice_update.agent_text_to_speech); + } + + #[test] + fn failed_disabled_voice_save_does_not_change_the_remembered_voice() { + let state = crate::app_state::build_app_state(); + state + .tts_settings + .lock() + .expect("settings") + .agent_text_to_speech = false; + let current = state.tts_settings.lock().expect("settings").clone(); + let unsaved = settings_with_pocket_voice_from_registry( + current, + MARIUS_VOICE_KEY, + &bundled_voice_registry(), + ) + .expect("available voice"); + + // This is the only pre-persistence mutation for an OFF candidate. + commit_effective_off(&state).expect("commit effective OFF state"); + let remembered = state.tts_settings.lock().expect("settings").clone(); + assert_eq!(remembered.voice_preferences, vec![MARY_VOICE_KEY]); + assert_eq!(unsaved.voice_preferences, vec![MARIUS_VOICE_KEY]); + } +} diff --git a/desktop/src-tauri/src/huddle/tts_voice_import.rs b/desktop/src-tauri/src/huddle/tts_voice_import.rs new file mode 100644 index 0000000000..1813b19ab3 --- /dev/null +++ b/desktop/src-tauri/src/huddle/tts_voice_import.rs @@ -0,0 +1,56 @@ +//! Tauri native-picker adapter for the reusable local Pocket voice library. + +use std::path::PathBuf; + +use buzz_voice_pkg::imported::{ImportedVoice, PocketVoiceLibrary}; +use tauri::{AppHandle, Manager}; + +pub fn voices_dir(app: &AppHandle) -> Result { + app.path() + .app_data_dir() + .map(|path| path.join("tts").join("pocket-voices")) + .map_err(|error| format!("could not locate local voice storage: {error}")) +} + +fn library(app: &AppHandle) -> Result { + voices_dir(app).map(PocketVoiceLibrary::new) +} + +pub fn load_registry(app: &AppHandle) -> Result, String> { + library(app)?.load() +} + +pub fn resolve_file(app: &AppHandle, voice: &ImportedVoice) -> Result { + library(app)?.resolve_file(voice) +} + +pub async fn pick_and_import(app: &AppHandle) -> Result, String> { + use tauri_plugin_dialog::DialogExt; + + let (sender, receiver) = tokio::sync::oneshot::channel(); + app.dialog() + .file() + .add_filter("WAV audio", &["wav"]) + .pick_file(move |path| { + let _ = sender.send(path); + }); + let Some(file_path) = receiver + .await + .map_err(|_| "voice picker closed unexpectedly".to_string())? + else { + return Ok(None); + }; + let path = file_path + .as_path() + .ok_or("the selected voice path is invalid")? + .to_path_buf(); + let voice_library = library(app)?; + tokio::task::spawn_blocking(move || voice_library.import_path(&path)) + .await + .map_err(|error| format!("voice import task failed: {error}"))? + .map(Some) +} + +pub fn delete(app: &AppHandle, key: &str) -> Result<(), String> { + library(app)?.delete(key) +} diff --git a/desktop/src-tauri/src/huddle/tts_voice_selection_tests.rs b/desktop/src-tauri/src/huddle/tts_voice_selection_tests.rs new file mode 100644 index 0000000000..04bccd4605 --- /dev/null +++ b/desktop/src-tauri/src/huddle/tts_voice_selection_tests.rs @@ -0,0 +1,338 @@ +use super::*; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +#[test] +fn selecting_a_voice_raises_only_the_internal_cancel_and_retains_the_engine_handle() { + let model_dir = tempfile::tempdir().expect("temp model dir"); + let active = Arc::new(AtomicBool::new(false)); + let cancel = Arc::new(AtomicBool::new(false)); + let pipeline = TtsPipeline::new_with_voice( + model_dir.path().to_path_buf(), + active, + Arc::clone(&cancel), + "reference_sample", + None, + ) + .expect("pipeline handle"); + + let _acknowledged = pipeline.select_voice("marius"); + + assert!(!cancel.load(Ordering::Acquire)); + assert!(pipeline.voice_cancel.load(Ordering::Acquire)); + assert_eq!( + pipeline + .voice + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_str(), + "marius" + ); +} + +#[test] +fn reconciling_an_unpublished_pipeline_does_not_cancel_its_first_message() { + let model_dir = tempfile::tempdir().expect("temp model dir"); + let active = Arc::new(AtomicBool::new(false)); + let cancel = Arc::new(AtomicBool::new(false)); + let pipeline = TtsPipeline::new_with_voice( + model_dir.path().to_path_buf(), + active, + Arc::clone(&cancel), + "reference_sample", + None, + ) + .expect("pipeline handle"); + + pipeline.select_voice_before_publish("marius"); + + assert!(!cancel.load(Ordering::Acquire)); + assert_eq!( + pipeline + .voice + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_str(), + "marius" + ); +} + +#[test] +fn received_text_reconciles_a_voice_changed_while_the_worker_was_waiting() { + let model_dir = tempfile::tempdir().expect("temp model dir"); + let bundled_voice = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("resources/pocket-voices/marius.wav"); + std::fs::copy( + &bundled_voice, + model_dir.path().join("reference_sample.wav"), + ) + .expect("Mary test voice"); + std::fs::copy(&bundled_voice, model_dir.path().join("marius.wav")).expect("Marius test voice"); + + let selected_voice = Arc::new(std::sync::Mutex::new("reference_sample".to_string())); + let mut style = + load_voice_style(&model_dir.path().join("reference_sample.wav")).expect("initial style"); + let waiting = Arc::new(std::sync::Barrier::new(2)); + let (text_tx, text_rx) = std::sync::mpsc::channel(); + let worker_voice = Arc::clone(&selected_voice); + let worker_waiting = Arc::clone(&waiting); + let worker_model_dir = model_dir.path().to_path_buf(); + let worker = std::thread::spawn(move || { + let mut voice_name = "reference_sample".to_string(); + worker_waiting.wait(); + let text = text_rx.recv().expect("first queued text"); + assert!(reconcile_selected_voice( + &worker_model_dir, + &worker_voice, + &mut voice_name, + &mut style, + )); + (text, voice_name) + }); + + waiting.wait(); + *selected_voice.lock().expect("selected voice") = "marius".to_string(); + text_tx + .send("first message".to_string()) + .expect("queue first message"); + + assert_eq!( + worker.join().expect("worker"), + ("first message".to_string(), "marius".to_string()) + ); +} + +#[test] +fn an_in_hand_post_change_message_survives_cancellation() { + let selected_voice = Arc::new(std::sync::Mutex::new("reference_sample".to_string())); + let voice_generation = AtomicU64::new(1); + let barge_in = AtomicBool::new(false); + let voice_cancel = Arc::new(AtomicBool::new(false)); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (text_tx, text_rx) = std::sync::mpsc::sync_channel(1); + let mut acknowledged = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "marius", + ) + .expect("voice changed"); + assert!(voice_cancel.load(Ordering::Acquire)); + assert!(matches!( + acknowledged.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + acknowledge_voice_change(&voice_change_ack, &voice_cancel); + assert!(matches!( + acknowledged.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + text_tx + .send(QueuedText { + generation: voice_generation.load(Ordering::Acquire), + text: "new message".to_string(), + }) + .expect("new message"); + let mut current_text = Some(text_rx.recv().expect("in-hand new message")); + + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let mut deferred_text = VecDeque::from([ + QueuedText { + generation: 1, + text: "old message".to_string(), + }, + QueuedText { + generation: voice_generation.load(Ordering::Acquire), + text: "later new message".to_string(), + }, + ]); + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + acknowledge_voice_change(&voice_change_ack, &voice_cancel); + acknowledged.blocking_recv().expect("voice change ack"); + + assert_eq!( + deferred_text + .pop_front() + .expect("preserved post-change message") + .text, + "new message" + ); + assert_eq!( + deferred_text + .pop_front() + .expect("later post-change message") + .text, + "later new message" + ); + assert!(text_rx.try_recv().is_err()); +} + +#[test] +fn superseding_voice_change_removes_earlier_deferred_messages() { + let selected_voice = std::sync::Mutex::new("reference_sample".to_string()); + let voice_generation = AtomicU64::new(1); + let barge_in = AtomicBool::new(false); + let voice_cancel = AtomicBool::new(false); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (_text_tx, text_rx) = std::sync::mpsc::channel(); + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let mut deferred_text = VecDeque::new(); + let mut current_text = None; + + let first = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "marius", + ) + .expect("first voice change"); + deferred_text.push_back(QueuedText { + generation: voice_generation.load(Ordering::Acquire), + text: "message for Marius".to_string(), + }); + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + acknowledge_voice_change(&voice_change_ack, &voice_cancel); + first.blocking_recv().expect("first acknowledgement"); + + let _second = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "reference_sample", + ) + .expect("second voice change"); + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + + assert!(deferred_text.is_empty()); +} + +#[test] +fn barge_in_clears_deferred_voice_change_messages() { + let barge_in = AtomicBool::new(true); + let voice_cancel = AtomicBool::new(false); + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (_text_tx, text_rx) = std::sync::mpsc::channel(); + let mut deferred_text = VecDeque::from([QueuedText { + generation: 2, + text: "deferred message".to_string(), + }]); + let mut current_text = None; + + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + + assert!(deferred_text.is_empty()); +} + +#[test] +fn barge_in_during_a_voice_change_clears_post_change_messages() { + let selected_voice = std::sync::Mutex::new("reference_sample".to_string()); + let voice_generation = AtomicU64::new(1); + let barge_in = AtomicBool::new(false); + let voice_cancel = AtomicBool::new(false); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (_text_tx, text_rx) = std::sync::mpsc::channel(); + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let mut deferred_text = VecDeque::new(); + let mut current_text = None; + + let _acknowledged = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "marius", + ) + .expect("voice change"); + deferred_text.push_back(QueuedText { + generation: voice_generation.load(Ordering::Acquire), + text: "post-change message".to_string(), + }); + barge_in.store(true, Ordering::Release); + + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + assert!(deferred_text.is_empty()); +} + +#[test] +fn a_sender_captured_before_voice_change_is_stale_even_if_it_sends_after_drain() { + let selected_voice = std::sync::Mutex::new("reference_sample".to_string()); + let voice_generation = Arc::new(AtomicU64::new(1)); + let barge_in = AtomicBool::new(false); + let voice_cancel = AtomicBool::new(false); + let voice_change_ack = Arc::new(std::sync::Mutex::new(None)); + let (text_tx, text_rx) = std::sync::mpsc::sync_channel(1); + let old_sender = TtsTextSender { + text_tx, + generation: voice_generation.load(Ordering::Acquire), + }; + let shutdown = AtomicBool::new(false); + let active = AtomicBool::new(true); + let mut deferred_text = VecDeque::new(); + let mut current_text = None; + + let _acknowledged = begin_voice_change( + &selected_voice, + &voice_generation, + &voice_cancel, + &voice_change_ack, + "marius", + ) + .expect("voice change"); + assert!(handle_cancel_or_shutdown( + (&barge_in, &voice_cancel), + &shutdown, + &active, + (&text_rx, &mut deferred_text, &mut current_text), + &voice_change_ack, + None, + )); + old_sender + .send("late old message".to_string()) + .expect("late send"); + let late = text_rx.recv().expect("late queued text"); + + assert!(late.generation < voice_generation.load(Ordering::Acquire)); +} diff --git a/desktop/src-tauri/src/huddle/tts_voice_transition.rs b/desktop/src-tauri/src/huddle/tts_voice_transition.rs new file mode 100644 index 0000000000..3fd3afb8f5 --- /dev/null +++ b/desktop/src-tauri/src/huddle/tts_voice_transition.rs @@ -0,0 +1,184 @@ +use std::{ + collections::VecDeque, + path::Path, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc::{self, SyncSender}, + Arc, Mutex, + }, +}; + +use crate::huddle::pocket::{load_voice_style, VoiceStyle, DEFAULT_VOICE, VOICE_FILE_EXT}; + +#[derive(Debug)] +pub(super) struct PendingVoiceChange { + pub(super) generation: u64, + acknowledged: tokio::sync::oneshot::Sender<()>, +} + +pub(super) type VoiceChangeAck = Arc>>; +pub(super) type WorkerVoiceState = (Arc>, Arc, VoiceChangeAck); +pub(super) type WorkerCancelSignals = (Arc, Arc); +pub(super) type CancelTextState<'a> = ( + &'a mpsc::Receiver, + &'a mut VecDeque, + &'a mut Option, +); +pub(super) type CancelSignals<'a> = (&'a AtomicBool, &'a AtomicBool); + +#[derive(Debug)] +pub(super) struct QueuedText { + pub(super) generation: u64, + pub(super) text: String, +} + +#[derive(Clone, Debug)] +pub(crate) struct TtsTextSender { + pub(super) text_tx: SyncSender, + pub(super) generation: u64, +} + +impl TtsTextSender { + pub(crate) fn send(&self, text: String) -> Result<(), String> { + self.text_tx + .send(QueuedText { + generation: self.generation, + text, + }) + .map_err(|error| error.to_string()) + } +} + +pub(super) fn begin_voice_change( + selected_voice: &Mutex, + voice_generation: &AtomicU64, + voice_cancel: &AtomicBool, + voice_change_ack: &VoiceChangeAck, + voice: &str, +) -> Option> { + let mut pending_ack = voice_change_ack + .lock() + .unwrap_or_else(|error| error.into_inner()); + let mut selected = selected_voice + .lock() + .unwrap_or_else(|error| error.into_inner()); + if selected.as_str() == voice { + return None; + } + + let (sender, receiver) = tokio::sync::oneshot::channel(); + voice_cancel.store(true, Ordering::Release); + let generation = voice_generation.fetch_add(1, Ordering::AcqRel) + 1; + if let Some(superseded) = pending_ack.replace(PendingVoiceChange { + generation, + acknowledged: sender, + }) { + let _ = superseded.acknowledged.send(()); + } + *selected = voice.to_string(); + Some(receiver) +} + +pub(super) fn acknowledge_voice_change( + voice_change_ack: &VoiceChangeAck, + voice_cancel: &AtomicBool, +) { + let mut pending_ack = voice_change_ack + .lock() + .unwrap_or_else(|error| error.into_inner()); + if voice_cancel.load(Ordering::Acquire) { + return; + } + if let Some(pending) = pending_ack.take() { + let _ = pending.acknowledged.send(()); + } +} + +pub(super) fn finish_voice_change_ack(voice_change_ack: &VoiceChangeAck) { + if let Some(pending) = voice_change_ack + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + let _ = pending.acknowledged.send(()); + } +} + +pub(super) fn reconcile_selected_voice( + model_dir: &Path, + selected_voice: &Mutex, + voice_name: &mut String, + style: &mut VoiceStyle, +) -> bool { + let requested_voice = selected_voice + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + if requested_voice == *voice_name { + return true; + } + + let requested_path = voice_path(model_dir, &requested_voice); + match load_voice_style(&requested_path) { + Ok(requested_style) => { + *style = requested_style; + *voice_name = requested_voice; + true + } + Err(error) => { + eprintln!( + "buzz-desktop: Pocket voice {requested_voice} is unavailable ({error}); falling back to Mary" + ); + let fallback_path = model_dir.join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}")); + match load_voice_style(&fallback_path) { + Ok(fallback_style) => { + *style = fallback_style; + *voice_name = DEFAULT_VOICE.to_string(); + *selected_voice + .lock() + .unwrap_or_else(|lock_error| lock_error.into_inner()) = + DEFAULT_VOICE.to_string(); + true + } + Err(fallback_error) => { + eprintln!("buzz-desktop: Mary voice fallback is unavailable: {fallback_error}"); + false + } + } + } + } +} + +pub(super) fn voice_path(model_dir: &Path, voice: &str) -> std::path::PathBuf { + let path = Path::new(voice); + if path.is_absolute() { + path.to_path_buf() + } else { + model_dir.join(format!("{voice}.{VOICE_FILE_EXT}")) + } +} + +pub(super) fn retain_cancelled_text( + deferred_text: &mut VecDeque, + current_text: &mut Option, + text_rx: &mpsc::Receiver, + preserve_generation: Option, +) { + if let Some(generation) = preserve_generation { + deferred_text.retain(|text| text.generation >= generation); + if let Some(text) = current_text.take() { + if text.generation >= generation { + deferred_text.push_front(text); + } + } + while let Ok(text) = text_rx.try_recv() { + if text.generation >= generation { + deferred_text.push_back(text); + } + } + } else { + deferred_text.clear(); + current_text.take(); + while text_rx.try_recv().is_ok() {} + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 5346791ccf..0bf6c277cc 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -460,6 +460,18 @@ pub fn run() { *guard = Some(app_handle.clone()); } + let (tts_settings, tts_settings_load_error) = + huddle::tts_settings::load_for_app(&app_handle); + if let Ok(mut guard) = state.tts_settings.lock() { + *guard = tts_settings.clone(); + } + if let Ok(mut guard) = state.tts_settings_load_error.lock() { + *guard = tts_settings_load_error; + } + if let Ok(mut huddle) = state.huddle_state.lock() { + huddle.tts_enabled = tts_settings.agent_text_to_speech; + } + // Bring up the runtime-owned shared-compute coordinator before // saved agents are restored. Its lifetime is tied to the app, not // a UI mount; it publishes discovery and reconciles membership for @@ -877,6 +889,12 @@ pub fn run() { download_voice_models, get_model_status, set_tts_enabled, + huddle::tts_settings::get_tts_settings, + huddle::tts_settings::list_voice_registry, + huddle::tts_settings::set_pocket_voice, + huddle::tts_settings::preview_pocket_voice, + huddle::tts_settings::import_pocket_voice, + huddle::tts_settings::delete_pocket_voice, speak_agent_message, add_agent_to_huddle, check_pipeline_hotstart, diff --git a/desktop/src/features/huddle/lib/ttsLiveMessages.test.mjs b/desktop/src/features/huddle/lib/ttsLiveMessages.test.mjs new file mode 100644 index 0000000000..088a6d6ebf --- /dev/null +++ b/desktop/src/features/huddle/lib/ttsLiveMessages.test.mjs @@ -0,0 +1,154 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + createInitialMembershipGate, + createLatestStateGate, + createOrderedSpeaker, + speakableAgentText, +} from "./ttsLiveMessages.ts"; + +const agents = new Set(["agent"]); +const base = { + id: "1", + kind: 9, + pubkey: "agent", + content: "Hello there", + tags: [], +}; + +test("speaks only new agent-authored text message events", () => { + assert.equal(speakableAgentText(base, agents, "human"), "Hello there"); + assert.equal( + speakableAgentText({ ...base, kind: 7 }, agents, "human"), + null, + "reactions and other event kinds are excluded", + ); + assert.equal( + speakableAgentText({ ...base, kind: 10 }, agents, "human"), + null, + "edits and status events are excluded", + ); + assert.equal( + speakableAgentText({ ...base, pubkey: "human" }, agents, "human"), + null, + "human-authored messages are excluded", + ); + assert.equal( + speakableAgentText({ ...base, content: " " }, agents, "human"), + null, + "empty and non-text content are excluded", + ); + assert.equal( + speakableAgentText( + { ...base, content: "[System] tool started" }, + agents, + "human", + ), + null, + "legacy system rows are excluded", + ); +}); + +test("strips attachment markup and skips attachment-only events", () => { + const url = "https://cdn.example/voice.png"; + const tags = [["imeta", `url ${url}`, "m image/png"]]; + assert.equal( + speakableAgentText( + { ...base, content: `![image](${url})`, tags }, + agents, + "human", + ), + null, + ); + assert.equal( + speakableAgentText( + { ...base, content: `Here is the diagram.\n\n![image](${url})`, tags }, + agents, + "human", + ), + "Here is the diagram.", + ); + assert.equal( + speakableAgentText( + { ...base, content: `||\n![image](${url})\n||`, tags }, + agents, + "human", + ), + null, + ); +}); + +test("queues agent messages in live thread arrival order", async () => { + const spoken = []; + let releaseFirst; + const firstBlocked = new Promise((resolve) => { + releaseFirst = resolve; + }); + const speaker = createOrderedSpeaker(async (text) => { + if (text === "first") await firstBlocked; + spoken.push(text); + }, assert.fail); + + speaker.enqueue("first"); + speaker.enqueue("second"); + await Promise.resolve(); + assert.deepEqual(spoken, []); + releaseFirst(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.deepEqual(spoken, ["first", "second"]); +}); + +test("disabling cancels queued speech and rejects new messages until enabled", async () => { + const invoked = []; + let releaseFirst; + const firstBlocked = new Promise((resolve) => { + releaseFirst = resolve; + }); + const speaker = createOrderedSpeaker(async (text) => { + invoked.push(text); + if (text === "first") await firstBlocked; + }, assert.fail); + + speaker.enqueue("first"); + speaker.enqueue("queued-before-off"); + await Promise.resolve(); + speaker.setEnabled(false); + speaker.enqueue("while-off"); + releaseFirst(); + await new Promise((resolve) => setTimeout(resolve, 0)); + speaker.setEnabled(true); + speaker.enqueue("after-on"); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.deepEqual(invoked, ["first", "after-on"]); +}); + +test("a live TTS state event supersedes a delayed bootstrap result", () => { + const applied = []; + const gate = createLatestStateGate((enabled) => applied.push(enabled)); + const applyBootstrap = gate.beginSnapshot(); + + gate.applyEvent(false); + applyBootstrap(true); + + assert.deepEqual(applied, [false]); +}); + +test("buffers initial live events until membership resolves in order", () => { + const delivered = []; + const gate = createInitialMembershipGate((event) => delivered.push(event)); + gate.push("first"); + gate.push("second"); + assert.deepEqual(delivered, []); + gate.succeed(); + gate.push("third"); + assert.deepEqual(delivered, ["first", "second", "third"]); +}); + +test("drops the initial buffer fail-closed when membership lookup fails", () => { + const delivered = []; + const gate = createInitialMembershipGate((event) => delivered.push(event)); + gate.push("unverified"); + gate.fail(); + assert.deepEqual(delivered, []); +}); diff --git a/desktop/src/features/huddle/lib/ttsLiveMessages.ts b/desktop/src/features/huddle/lib/ttsLiveMessages.ts new file mode 100644 index 0000000000..1e1aeb8b3f --- /dev/null +++ b/desktop/src/features/huddle/lib/ttsLiveMessages.ts @@ -0,0 +1,124 @@ +export type LiveTtsEvent = { + id: string; + kind: number; + pubkey: string; + content: string; + tags: string[][]; +}; + +function textWithoutAttachments(event: LiveTtsEvent): string { + const urls = new Set( + event.tags + .filter((tag) => tag[0] === "imeta") + .flatMap((tag) => + tag + .slice(1) + .filter((field) => field.startsWith("url ")) + .map((field) => field.slice(4)), + ), + ); + if (urls.size === 0) return event.content; + const withoutMedia = event.content + .split("\n") + .filter( + (line) => !Array.from(urls).some((url) => line.includes(`](${url})`)), + ) + .join("\n"); + return withoutMedia.replace( + /(^|\n)\s*\|\|\s*\n(?:\s*\n)*\s*\|\|\s*(?=\n|$)/gu, + "$1", + ); +} + +export function speakableAgentText( + event: LiveTtsEvent, + agentPubkeys: ReadonlySet, + selfPubkey: string | null, +): string | null { + if (event.kind !== 9) return null; + if (!agentPubkeys.has(event.pubkey)) return null; + if (event.pubkey === selfPubkey) return null; + const content = textWithoutAttachments(event).trim(); + if (content.length <= 1) return null; + if (content.startsWith("[System]")) return null; + return content; +} + +/** + * Serialize native speak calls so live messages enter the bounded Pocket queue + * in thread arrival order even when the bridge resolves calls asynchronously. + */ +export function createOrderedSpeaker( + speak: (text: string) => Promise, + onError: (error: unknown) => void, +): { + enqueue: (text: string) => void; + setEnabled: (enabled: boolean) => void; +} { + let tail = Promise.resolve(); + let enabled = true; + let generation = 0; + return { + enqueue(text) { + if (!enabled) return; + const queuedGeneration = generation; + tail = tail + .then(() => { + if (!enabled || generation !== queuedGeneration) return; + return speak(text); + }) + .catch(onError); + }, + setEnabled(nextEnabled) { + if (!nextEnabled) generation += 1; + enabled = nextEnabled; + }, + }; +} + +/** Ensure a delayed bootstrap snapshot cannot overwrite a newer live event. */ +export function createLatestStateGate(apply: (value: T) => void): { + applyEvent: (value: T) => void; + beginSnapshot: () => (value: T) => void; +} { + let revision = 0; + return { + applyEvent(value) { + revision += 1; + apply(value); + }, + beginSnapshot() { + const snapshotRevision = revision; + return (value) => { + if (revision === snapshotRevision) apply(value); + }; + }, + }; +} + +/** Hold live events until the first authoritative agent-membership lookup. */ +export function createInitialMembershipGate(deliver: (event: T) => void): { + push: (event: T) => void; + succeed: () => void; + fail: () => void; +} { + let settled = false; + let pending: T[] = []; + return { + push(event) { + if (settled) deliver(event); + else pending.push(event); + }, + succeed() { + if (settled) return; + settled = true; + const buffered = pending; + pending = []; + for (const event of buffered) deliver(event); + }, + fail() { + settled = true; + pending = []; + }, + }; +} diff --git a/desktop/src/features/huddle/lib/useTtsSubscription.ts b/desktop/src/features/huddle/lib/useTtsSubscription.ts index d534fc0166..8c6d8f67b5 100644 --- a/desktop/src/features/huddle/lib/useTtsSubscription.ts +++ b/desktop/src/features/huddle/lib/useTtsSubscription.ts @@ -1,7 +1,15 @@ import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; import * as React from "react"; +import { buildHuddleTtsLiveFilter } from "@/shared/api/relayChannelFilters"; import { relayClient } from "@/shared/api/relayClient"; +import { + createInitialMembershipGate, + createLatestStateGate, + createOrderedSpeaker, + speakableAgentText, +} from "./ttsLiveMessages"; const AGENT_PUBKEY_REFRESH_INTERVAL_MS = 30_000; @@ -20,6 +28,7 @@ export function useTtsSubscription( let disposed = false; let cleanup: (() => void) | null = null; + let unlistenHuddleState: (() => void) | null = null; // ── Agent identity (authoritative, fail-closed) ─────────────────────── // @@ -33,66 +42,115 @@ export function useTtsSubscription( let agentsLoaded = false; const agentPubkeys = new Set(); - async function loadAgentPubkeys() { + const speakInOrder = createOrderedSpeaker( + async (text) => { + if (!disposed) { + await invoke("speak_agent_message", { text }); + } + }, + (err) => { + console.warn("[huddle] TTS speak failed:", err); + }, + ); + + const deliver = (event: Parameters[0]) => { + if (!agentsLoaded || disposed) return; + const text = speakableAgentText( + event, + agentPubkeys, + selfPubkeyRef.current, + ); + if (text) speakInOrder.enqueue(text); + }; + const initialMembershipGate = createInitialMembershipGate(deliver); + + async function loadAgentPubkeys(initial = false) { try { const pubkeys = await invoke("get_huddle_agent_pubkeys"); + if (disposed) return; agentPubkeys.clear(); for (const pk of pubkeys) agentPubkeys.add(pk); agentsLoaded = true; + if (initial) { + initialMembershipGate.succeed(); + } } catch (e) { // Fail-closed on ALL failures, including refresh after prior success. // Clear the set and mark as not loaded — TTS goes mute until the // next successful refresh. Stale membership must never authorize speech. agentPubkeys.clear(); agentsLoaded = false; + if (initial) { + initialMembershipGate.fail(); + } console.error("[huddle] Failed to load agent pubkeys:", e); } } // Initial load + periodic refresh (catches mid-huddle agent additions). - void loadAgentPubkeys(); + void loadAgentPubkeys(true); const agentRefreshId = window.setInterval(() => { void loadAgentPubkeys(); }, AGENT_PUBKEY_REFRESH_INTERVAL_MS); + // Install the state listener before requesting a snapshot. If a newer + // event arrives while IPC is pending, it supersedes the stale snapshot. + const ttsStateGate = createLatestStateGate<{ tts_enabled: boolean }>( + (state) => { + if (!disposed) speakInOrder.setEnabled(state.tts_enabled); + }, + ); + void listen<{ tts_enabled: boolean }>("huddle-state-changed", (event) => { + if (!disposed) ttsStateGate.applyEvent(event.payload); + }) + .then((unlisten) => { + if (disposed) { + unlisten(); + return; + } + unlistenHuddleState = unlisten; + const applyBootstrap = ttsStateGate.beginSnapshot(); + void invoke<{ tts_enabled: boolean }>("get_huddle_state") + .then((state) => { + if (!disposed) applyBootstrap(state); + }) + .catch((err) => { + if (!disposed) applyBootstrap({ tts_enabled: false }); + console.warn("[huddle] Failed to load TTS state:", err); + }); + }) + .catch((err) => { + speakInOrder.setEnabled(false); + console.warn("[huddle] Failed to listen for TTS state:", err); + }); + // ── Live-only subscription ─────────────────────────────────────────── - // subscribeToChannelLive uses `since: now` — the relay never sends - // historical backlog. Every event delivered is a live message. + // A kind:9, limit:0 subscription receives future fan-out while the relay + // returns no stored rows, including pre-join rows from the current second. // Event-ID dedup handles reconnect replay (same event arriving twice). const seenEventIds = new Set(); const seenOrder: string[] = []; const MAX_SEEN_EVENTS = 5000; - relayClient - .subscribeToChannelLive(ephemeralChannelId, (event) => { - if (disposed) return; - // Defense-in-depth: subscription already filters to kind:9 only. - if (event.kind !== 9) return; - - // Dedup by event ID (covers reconnect replay). - if (seenEventIds.has(event.id)) return; - seenEventIds.add(event.id); - seenOrder.push(event.id); - if (seenOrder.length > MAX_SEEN_EVENTS) { - const oldest = seenOrder.shift(); - if (oldest !== undefined) seenEventIds.delete(oldest); - } + .subscribeLive( + buildHuddleTtsLiveFilter(ephemeralChannelId), + (event) => { + if (disposed) return; + // Dedup by event ID (covers reconnect replay). + if (seenEventIds.has(event.id)) return; + seenEventIds.add(event.id); + seenOrder.push(event.id); + if (seenOrder.length > MAX_SEEN_EVENTS) { + const oldest = seenOrder.shift(); + if (oldest !== undefined) seenEventIds.delete(oldest); + } - // Fail-closed: don't speak until agent list is loaded. - if (!agentsLoaded) return; - // Only speak agent messages — skip human STT transcripts. - if (!agentPubkeys.has(event.pubkey)) return; - if (event.pubkey === selfPubkeyRef.current) return; - if (event.content.trim().length <= 1) return; - // Legacy: skip [System]-prefixed messages from before kind:48106. - if (event.content.startsWith("[System]")) return; - invoke("speak_agent_message", { text: event.content }).catch((err) => { - console.warn( - "[huddle] TTS speak failed (backpressure or pipeline unavailable):", - err, - ); - }); - }) + // Preserve arrival order while the initial authoritative membership + // lookup is pending. A failed lookup clears this buffer fail-closed. + initialMembershipGate.push(event); + }, + { replayMissedHistory: true }, + ) .then((dispose) => { if (disposed) { void dispose(); @@ -106,7 +164,9 @@ export function useTtsSubscription( return () => { disposed = true; + speakInOrder.setEnabled(false); cleanup?.(); + unlistenHuddleState?.(); window.clearInterval(agentRefreshId); }; }, [ephemeralChannelId, selfPubkeyRef]); diff --git a/desktop/src/features/settings/ui/SettingsPanels.tsx b/desktop/src/features/settings/ui/SettingsPanels.tsx index 156be00b72..27a0674f80 100644 --- a/desktop/src/features/settings/ui/SettingsPanels.tsx +++ b/desktop/src/features/settings/ui/SettingsPanels.tsx @@ -21,6 +21,7 @@ import { SunMoon, Ticket, UserRound, + Volume2, type LucideIcon, } from "lucide-react"; import type { @@ -84,10 +85,12 @@ import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; import { ProfileSettingsCard } from "./ProfileSettingsCard"; import { UpdateChecker } from "../UpdateChecker"; import { SettingsSectionHeader } from "./SettingsSectionHeader"; +import { VoiceSettingsCard } from "./VoiceSettingsCard"; export type SettingsSection = | "profile" | "notifications" + | "voice" | "experimental" | "agents" | "channel-templates" @@ -107,6 +110,7 @@ export const DEFAULT_SETTINGS_SECTION: SettingsSection = "profile"; const SETTINGS_SECTION_VALUES: readonly SettingsSection[] = [ "profile", "notifications", + "voice", "experimental", "agents", "channel-templates", @@ -168,6 +172,11 @@ export const settingsSections: SettingsSectionDescriptor[] = [ label: "Notifications", icon: BellRing, }, + { + value: "voice", + label: "Voice", + icon: Volume2, + }, { value: "experimental", label: "Experiments", @@ -808,6 +817,8 @@ export function renderSettingsSection( onSetSoundForSlot={props.onSetSoundForSlot} /> ); + case "voice": + return ; case "experimental": return ; case "agents": diff --git a/desktop/src/features/settings/ui/SettingsView.tsx b/desktop/src/features/settings/ui/SettingsView.tsx index 20389b420a..8613880571 100644 --- a/desktop/src/features/settings/ui/SettingsView.tsx +++ b/desktop/src/features/settings/ui/SettingsView.tsx @@ -58,6 +58,7 @@ const settingsNavGroups: Array<{ "profile", "appearance", "notifications", + "voice", "shortcuts", "custom-emoji", "local-archive", diff --git a/desktop/src/features/settings/ui/VoiceSettingsCard.tsx b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx new file mode 100644 index 0000000000..8c910c190a --- /dev/null +++ b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx @@ -0,0 +1,370 @@ +import * as React from "react"; +import { ChevronDown, Play, Trash2, Upload, Volume2 } from "lucide-react"; + +import { invokeTauri } from "@/shared/api/tauri"; +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/shared/ui/alert-dialog"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuTrigger, +} from "@/shared/ui/dropdown-menu"; +import { Switch } from "@/shared/ui/switch"; +import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; +import { SettingsSectionHeader } from "./SettingsSectionHeader"; +import { + selectedVoiceForBackend, + type VoiceRegistryEntry, + voiceOptionLabel, + voicesForBackend, +} from "./voiceSettingsLogic"; + +export type TtsSettings = { + version: number; + agentTextToSpeech: boolean; + voicePreferences: string[]; +}; + +type TtsVoiceMutation = { + settings: TtsSettings; + registry: VoiceRegistryEntry[]; +}; + +export function VoiceSettingsCard() { + const [settings, setSettings] = React.useState(null); + const [registry, setRegistry] = React.useState([]); + const [busy, setBusy] = React.useState(false); + const [previewing, setPreviewing] = React.useState(false); + const [deleteCandidate, setDeleteCandidate] = + React.useState(null); + const [error, setError] = React.useState(null); + + React.useEffect(() => { + let disposed = false; + Promise.all([ + invokeTauri("get_tts_settings"), + invokeTauri("list_voice_registry"), + ]) + .then(([nextSettings, nextRegistry]) => { + if (!disposed) { + setSettings(nextSettings); + setRegistry(nextRegistry); + } + }) + .catch((loadError) => { + if (!disposed) { + setError( + loadError instanceof Error + ? loadError.message + : "Voice settings could not be loaded.", + ); + } + }); + return () => { + disposed = true; + }; + }, []); + + const saveEnabled = React.useCallback(async (enabled: boolean) => { + setBusy(true); + setError(null); + try { + const saved = await invokeTauri("set_tts_enabled", { + enabled, + }); + setSettings(saved); + } catch (saveError) { + try { + const state = await invokeTauri<{ tts_enabled: boolean }>( + "get_huddle_state", + ); + setSettings((current) => + current + ? { ...current, agentTextToSpeech: state.tts_enabled } + : current, + ); + } catch { + // Keep the last confirmed state when native reconciliation is + // unavailable; the visible save error makes the failure explicit. + } + setError( + saveError instanceof Error + ? saveError.message + : "Voice settings could not be saved.", + ); + } finally { + setBusy(false); + } + }, []); + + const savePocketVoice = React.useCallback(async (voiceKey: string) => { + setBusy(true); + setError(null); + try { + const saved = await invokeTauri("set_pocket_voice", { + voiceKey, + }); + setSettings(saved); + } catch (saveError) { + setError( + saveError instanceof Error + ? saveError.message + : "Voice settings could not be saved.", + ); + } finally { + setBusy(false); + } + }, []); + + const importPocketVoice = React.useCallback(async () => { + setBusy(true); + setError(null); + try { + const result = await invokeTauri( + "import_pocket_voice", + ); + if (result) { + setSettings(result.settings); + setRegistry(result.registry); + } + } catch (importError) { + setError( + importError instanceof Error + ? importError.message + : "Voice could not be imported.", + ); + } finally { + setBusy(false); + } + }, []); + + const deletePocketVoice = React.useCallback(async (voiceKey: string) => { + setBusy(true); + setError(null); + try { + const result = await invokeTauri( + "delete_pocket_voice", + { voiceKey }, + ); + setSettings(result.settings); + setRegistry(result.registry); + setDeleteCandidate(null); + } catch (deleteError) { + setError( + deleteError instanceof Error + ? deleteError.message + : "Voice could not be deleted.", + ); + } finally { + setBusy(false); + } + }, []); + + const voices = voicesForBackend(registry, "pocket"); + const selectedVoice = selectedVoiceForBackend( + settings?.voicePreferences ?? [], + voices, + ); + const enabled = settings?.agentTextToSpeech ?? true; + const controlsDisabled = !settings || busy || !enabled; + + return ( +
+ + +
+ + +
+ +

+ Read new agent messages aloud in the order they arrive. +

+
+ { + if (settings) void saveEnabled(checked); + }} + /> + + + +
+ + +
+

Voice

+

+ Voice files stay private on this device. +

+
+ +
+ + + + + + { + if (settings) void savePocketVoice(voiceKey); + }} + value={selectedVoice?.key} + > + {voices.map((voice) => ( + + {voiceOptionLabel(voice, voices)} + + ))} + + + + + + {selectedVoice?.key.startsWith("pocket:imported:") && ( + + )} +
+ + +
+ + {error && ( +

+ {error} +

+ )} +
+ { + if (!open) setDeleteCandidate(null); + }} + open={deleteCandidate !== null} + > + + + Delete imported voice? + + {deleteCandidate + ? `${deleteCandidate.displayName} and its local audio file will be removed.` + : "This imported voice and its local audio file will be removed."} + {selectedVoice?.key === deleteCandidate?.key && + " Mary will be selected instead."} + + + + Cancel + { + event.preventDefault(); + if (deleteCandidate) { + void deletePocketVoice(deleteCandidate.key); + } + }} + > + Delete voice + + + + +
+ ); +} diff --git a/desktop/src/features/settings/ui/voiceSettingsLogic.test.mjs b/desktop/src/features/settings/ui/voiceSettingsLogic.test.mjs new file mode 100644 index 0000000000..d9e5ce9d03 --- /dev/null +++ b/desktop/src/features/settings/ui/voiceSettingsLogic.test.mjs @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + selectedVoiceForBackend, + voiceOptionLabel, + voicesForBackend, +} from "./voiceSettingsLogic.ts"; + +const voice = (key, displayName, fallbackKey = "pocket:mary") => ({ + key, + displayName, + backend: "pocket", + backendName: "Pocket TTS", + availability: "bundled", + fallbackKey, + referenceFile: `${key}.wav`, + provenance: { + source: "bundled", + contentHash: null, + license: null, + sourceUrl: null, + }, +}); + +test("Pocket-only V1 filters the shared registry by backend", () => { + const registry = [ + voice("pocket:mary", "Mary", null), + { ...voice("siri:aaron", "Aaron"), backend: "siri" }, + ]; + assert.deepEqual( + voicesForBackend(registry, "pocket").map((entry) => entry.key), + ["pocket:mary"], + ); +}); + +test("local selection uses the first compatible qualified preference", () => { + const voices = [ + voice("pocket:mary", "Mary", null), + voice("pocket:marius", "Marius"), + ]; + assert.equal( + selectedVoiceForBackend( + ["siri:aaron", "pocket:marius", "pocket:mary"], + voices, + )?.key, + "pocket:marius", + ); +}); + +test("duplicate display labels remain distinct by content-derived key", () => { + const voices = [ + voice("pocket:imported:aaa", "Jim"), + voice("pocket:imported:bbb", "Jim"), + ]; + assert.equal( + selectedVoiceForBackend(["pocket:imported:bbb"], voices)?.key, + "pocket:imported:bbb", + ); + assert.equal(voiceOptionLabel(voices[0], voices), "Jim · aaa"); + assert.equal(voiceOptionLabel(voices[1], voices), "Jim · bbb"); +}); diff --git a/desktop/src/features/settings/ui/voiceSettingsLogic.ts b/desktop/src/features/settings/ui/voiceSettingsLogic.ts new file mode 100644 index 0000000000..30352f2d0f --- /dev/null +++ b/desktop/src/features/settings/ui/voiceSettingsLogic.ts @@ -0,0 +1,57 @@ +export type VoiceAvailability = + | "bundled" + | "installed" + | "downloadable" + | "unavailable"; + +export type VoiceRegistryEntry = { + key: string; + displayName: string; + backend: string; + backendName: string; + availability: VoiceAvailability; + fallbackKey: string | null; + referenceFile: string | null; + provenance: { + source: string; + contentHash: string | null; + license: string | null; + sourceUrl: string | null; + }; +}; + +export function voicesForBackend( + registry: readonly VoiceRegistryEntry[], + backend: string, +): VoiceRegistryEntry[] { + return registry.filter( + (voice) => + voice.backend === backend && + (voice.availability === "bundled" || voice.availability === "installed"), + ); +} + +export function selectedVoiceForBackend( + preferences: readonly string[], + voices: readonly VoiceRegistryEntry[], +): VoiceRegistryEntry | undefined { + for (const key of preferences) { + const voice = voices.find((candidate) => candidate.key === key); + if (voice) return voice; + } + return voices.find((voice) => voice.fallbackKey === null) ?? voices[0]; +} + +export function voiceOptionLabel( + voice: VoiceRegistryEntry, + voices: readonly VoiceRegistryEntry[], +): string { + const duplicateLabel = voices.some( + (candidate) => + candidate.key !== voice.key && + candidate.displayName === voice.displayName, + ); + if (!duplicateLabel) return voice.displayName; + const identitySuffix = voice.key.split(":").at(-1)?.slice(-8) ?? voice.key; + return `${voice.displayName} · ${identitySuffix}`; +} diff --git a/desktop/src/shared/api/relayChannelFilters.test.mjs b/desktop/src/shared/api/relayChannelFilters.test.mjs index 3519c82146..7b89799049 100644 --- a/desktop/src/shared/api/relayChannelFilters.test.mjs +++ b/desktop/src/shared/api/relayChannelFilters.test.mjs @@ -6,6 +6,7 @@ import { buildChannelAuxFilter, buildChannelReactionAuxFilter, buildChannelStructuralAuxFilter, + buildHuddleTtsLiveFilter, } from "./relayChannelFilters.ts"; const CHANNEL = "36411e44-0e2d-4cfe-bd6e-567eb169db9f"; @@ -14,6 +15,14 @@ const IDS = [ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", ]; +test("huddle TTS filter is future-only kind:9 with no same-second replay", () => { + assert.deepEqual(buildHuddleTtsLiveFilter(CHANNEL), { + kinds: [9], + "#h": [CHANNEL], + limit: 0, + }); +}); + // Regression: reaction (kind:7) and reaction-removal (kind:5) events carry only // an `e` tag, no channel `h` tag. An `#h`-scoped aux query never matches them, // so removed historical reactions reappear. The aux filters must key on `#e` diff --git a/desktop/src/shared/api/relayChannelFilters.ts b/desktop/src/shared/api/relayChannelFilters.ts index d0c7e7938e..9acaef45c5 100644 --- a/desktop/src/shared/api/relayChannelFilters.ts +++ b/desktop/src/shared/api/relayChannelFilters.ts @@ -6,6 +6,7 @@ import { KIND_DELETION, KIND_NIP29_DELETE_EVENT, KIND_REACTION, + KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_EDIT, } from "@/shared/constants/kinds"; import type { RelaySubscriptionFilter } from "@/shared/api/relayClientShared"; @@ -40,6 +41,17 @@ export function buildChannelFilter( return filter; } +/** Strictly live huddle message filter: zero stored rows, future kind:9 only. */ +export function buildHuddleTtsLiveFilter( + channelId: string, +): RelaySubscriptionFilter { + return { + kinds: [KIND_STREAM_MESSAGE], + "#h": [channelId], + limit: 0, + }; +} + /** * History filter for cold-load and scrollback: message kinds *only*, so the * `limit` budget buys visible message depth. Auxiliary events (reactions, diff --git a/desktop/src/shared/api/relayClientSession.ts b/desktop/src/shared/api/relayClientSession.ts index 84ee10b68d..6be7fe04af 100644 --- a/desktop/src/shared/api/relayClientSession.ts +++ b/desktop/src/shared/api/relayClientSession.ts @@ -445,8 +445,9 @@ export class RelayClient { async subscribeLive( filter: RelaySubscriptionFilter, onEvent: (event: RelayEvent) => void, + options?: { replayMissedHistory?: boolean }, ) { - return this.subscribe(filter, onEvent); + return this.subscribe(filter, onEvent, options); } async subscribeToChannelMentionEvents( @@ -600,6 +601,7 @@ export class RelayClient { private async subscribe( filter: RelaySubscriptionFilter, onEvent: (event: RelayEvent) => void, + options?: { replayMissedHistory?: boolean }, ) { await this.ensureConnected(); @@ -621,6 +623,10 @@ export class RelayClient { mode: "live", filter, onEvent, + replayMissedHistory: options?.replayMissedHistory, + lastSeenCreatedAt: options?.replayMissedHistory + ? Math.floor(Date.now() / 1_000) + : undefined, resolveReady, }); diff --git a/desktop/src/shared/api/relayClientShared.ts b/desktop/src/shared/api/relayClientShared.ts index 9108f7b6d7..7559478b3d 100644 --- a/desktop/src/shared/api/relayClientShared.ts +++ b/desktop/src/shared/api/relayClientShared.ts @@ -58,6 +58,7 @@ type LiveSubscription = { mode: "live"; filter: RelaySubscriptionFilter; onEvent: (event: RelayEvent) => void; + replayMissedHistory?: boolean; resolveReady?: () => void; lastSeenCreatedAt?: number; closedRetryAttempt?: number; diff --git a/desktop/src/shared/api/relayReconnectReplay.test.mjs b/desktop/src/shared/api/relayReconnectReplay.test.mjs index 54254e89de..922d8ade49 100644 --- a/desktop/src/shared/api/relayReconnectReplay.test.mjs +++ b/desktop/src/shared/api/relayReconnectReplay.test.mjs @@ -5,6 +5,7 @@ import { buildReconnectReplayFilter, replayLiveSubscriptions, REPLAY_BATCH_SIZE, + shouldPageReconnectReplay, } from "./relayReconnectReplay.ts"; import { buildChannelFilter } from "./relayChannelFilters.ts"; @@ -113,6 +114,32 @@ test("reconnect replay caps large steady-state limits", () => { }); }); +test("reconnect replay preserves the live-only zero-history contract", () => { + const filter = { + kinds: [9], + "#h": ["channel-1"], + limit: 0, + }; + + assert.deepEqual(replayFilter(filter, 123), { + kinds: [9], + "#h": ["channel-1"], + limit: 0, + since: 123, + }); +}); + +test("missed-history replay is explicit for live-only subscriptions", () => { + const filter = { + kinds: [9], + "#h": ["channel-1"], + limit: 0, + }; + + assert.equal(shouldPageReconnectReplay(filter), false); + assert.equal(shouldPageReconnectReplay(filter, true), true); +}); + test("reconnect replay keeps the stricter existing since window", () => { const filter = { kinds: [9], diff --git a/desktop/src/shared/api/relayReconnectReplay.ts b/desktop/src/shared/api/relayReconnectReplay.ts index 74b752f666..6a09d1c549 100644 --- a/desktop/src/shared/api/relayReconnectReplay.ts +++ b/desktop/src/shared/api/relayReconnectReplay.ts @@ -69,7 +69,11 @@ export function buildReconnectReplayFilter( return replayFilter; } -export function shouldPageReconnectReplay(filter: RelaySubscriptionFilter) { +export function shouldPageReconnectReplay( + filter: RelaySubscriptionFilter, + replayMissedHistory = false, +) { + if (replayMissedHistory) return true; return ( filter.limit > 0 && Array.isArray(filter["#h"]) && @@ -176,7 +180,10 @@ export async function replayLiveSubscriptions({ ); const shouldPageReplay = replaySince !== undefined && - shouldPageReconnectReplay(subscription.filter); + shouldPageReconnectReplay( + subscription.filter, + subscription.replayMissedHistory, + ); return { subId, subscription, replaySince, shouldPageReplay }; }); diff --git a/desktop/src/testing/e2eBridge.ts b/desktop/src/testing/e2eBridge.ts index 03c05fe877..213c14e732 100644 --- a/desktop/src/testing/e2eBridge.ts +++ b/desktop/src/testing/e2eBridge.ts @@ -141,6 +141,13 @@ type MockSearchProfileSeed = { type E2eConfig = { mode?: "mock" | "relay"; mock?: { + ttsSettings?: { + version: number; + agentTextToSpeech: boolean; + voicePreferences: string[]; + }; + /** Native picker boundary result for Pocket voice import tests. */ + pocketVoiceImportResult?: "success" | "cancel" | "invalid"; /** Advertised HEAD for the first mock project without adding that branch. */ projectHeadBranch?: string; /** Builderlab account returned by hosted-community onboarding. Null/omitted = signed out. */ @@ -9412,7 +9419,25 @@ export function maybeInstallE2eTauriMocks() { deviceId: state === "running" ? "mock-endpoint-id" : null, deviceName: state === "running" ? "Mock desktop" : null, }); - const handleMockCommand = async (command: string, payload: unknown) => { + let mockImportedVoices: Array<{ + key: string; + displayName: string; + backend: string; + backendName: string; + availability: "installed"; + fallbackKey: string; + referenceFile: string; + provenance: { + source: string; + contentHash: string; + license: null; + sourceUrl: null; + }; + }> = []; + const handleMockCommand = async ( + command: string, + payload: unknown, + ): Promise => { const activeConfig = getConfig(); const identity = getActiveIdentity(activeConfig); window.__BUZZ_E2E_COMMANDS__?.push(command); @@ -9430,6 +9455,164 @@ export function maybeInstallE2eTauriMocks() { window.__BUZZ_E2E_COMMAND_LOG__?.push({ command, payload }); switch (command) { + case "get_tts_settings": + return ( + activeConfig?.mock?.ttsSettings ?? { + version: 1, + agentTextToSpeech: true, + voicePreferences: ["pocket:mary"], + } + ); + case "list_voice_registry": + return [ + { + key: "pocket:mary", + displayName: "Mary", + backend: "pocket", + backendName: "Pocket TTS", + availability: "bundled", + fallbackKey: null, + referenceFile: "reference_sample.wav", + provenance: { + source: "bundled", + contentHash: + "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f", + license: "CC-BY-4.0", + sourceUrl: "https://datashare.ed.ac.uk/handle/10283/3443", + }, + }, + { + key: "pocket:marius", + displayName: "Marius", + backend: "pocket", + backendName: "Pocket TTS", + availability: "bundled", + fallbackKey: "pocket:mary", + referenceFile: "marius.wav", + provenance: { + source: "bundled", + contentHash: + "076968c3122520f3412eb7090e8c1c3f75fe57be1e24a2f96465583d84c71e16", + license: "CC0-1.0", + sourceUrl: + "https://huggingface.co/kyutai/tts-voices/blob/323332d33f997de8394f24a193e1a76df720e01a/voice-donations/Selfie.wav", + }, + }, + ...mockImportedVoices, + ]; + case "set_tts_enabled": { + const enabled = (payload as { enabled?: boolean })?.enabled; + if (typeof enabled !== "boolean") + throw new Error("Missing text-to-speech enabled state"); + const settings = { + version: 1, + agentTextToSpeech: enabled, + voicePreferences: activeConfig?.mock?.ttsSettings + ?.voicePreferences ?? ["pocket:mary"], + }; + if (activeConfig) { + activeConfig.mock ??= {}; + activeConfig.mock.ttsSettings = settings; + } + return settings; + } + case "set_pocket_voice": { + const voiceKey = (payload as { voiceKey?: string })?.voiceKey; + if (!voiceKey) throw new Error("Missing Pocket voice key"); + const current = activeConfig?.mock?.ttsSettings ?? { + version: 1, + agentTextToSpeech: true, + voicePreferences: ["pocket:mary"], + }; + const firstPocketIndex = current.voicePreferences.findIndex((key) => + key.startsWith("pocket:"), + ); + const preferences = current.voicePreferences.filter( + (key) => !key.startsWith("pocket:"), + ); + preferences.splice( + firstPocketIndex < 0 ? preferences.length : firstPocketIndex, + 0, + voiceKey, + ); + const settings = { ...current, voicePreferences: preferences }; + if (activeConfig) { + activeConfig.mock ??= {}; + activeConfig.mock.ttsSettings = settings; + } + return settings; + } + case "preview_pocket_voice": + return null; + case "import_pocket_voice": { + const importResult = + activeConfig?.mock?.pocketVoiceImportResult ?? "success"; + if (importResult === "cancel") return null; + if (importResult === "invalid") { + throw new Error("Voice WAV must contain PCM or 32-bit float audio"); + } + const contentHash = "1".repeat(64); + const imported = { + key: `pocket:imported:${contentHash}`, + displayName: "My voice", + backend: "pocket", + backendName: "Pocket TTS", + availability: "installed" as const, + fallbackKey: "pocket:mary", + referenceFile: `${contentHash}.wav`, + provenance: { + source: "local import", + contentHash, + license: null, + sourceUrl: null, + }, + }; + mockImportedVoices = [imported]; + const current = activeConfig?.mock?.ttsSettings ?? { + version: 1, + agentTextToSpeech: true, + voicePreferences: ["pocket:mary"], + }; + const settings = { + ...current, + voicePreferences: [imported.key], + }; + if (activeConfig) { + activeConfig.mock ??= {}; + activeConfig.mock.ttsSettings = settings; + } + return { + settings, + registry: await handleMockCommand("list_voice_registry", null), + }; + } + case "delete_pocket_voice": { + const voiceKey = (payload as { voiceKey?: string })?.voiceKey; + if (!voiceKey?.startsWith("pocket:imported:")) + throw new Error("Missing imported Pocket voice key"); + mockImportedVoices = mockImportedVoices.filter( + (voice) => voice.key !== voiceKey, + ); + const current = activeConfig?.mock?.ttsSettings ?? { + version: 1, + agentTextToSpeech: true, + voicePreferences: ["pocket:mary"], + }; + const settings = { + ...current, + voicePreferences: current.voicePreferences.includes(voiceKey) + ? ["pocket:mary"] + : current.voicePreferences, + }; + if (activeConfig) { + activeConfig.mock ??= {}; + activeConfig.mock.ttsSettings = settings; + } + return { + settings, + registry: await handleMockCommand("list_voice_registry", null), + }; + } case "get_builderlab_auth": return activeConfig?.mock?.builderlabAuth ?? null; case "start_builderlab_login": { diff --git a/desktop/tests/e2e/voice-settings.spec.ts b/desktop/tests/e2e/voice-settings.spec.ts new file mode 100644 index 0000000000..da7db18175 --- /dev/null +++ b/desktop/tests/e2e/voice-settings.spec.ts @@ -0,0 +1,174 @@ +import { expect, test } from "@playwright/test"; + +import { waitForAnimations } from "../helpers/animations"; +import { installMockBridge } from "../helpers/bridge"; +import { openSettings } from "../helpers/settings"; + +const SCREENSHOT_PATH = "test-results/voice-settings/pocket-voices.png"; + +test.describe("Pocket voice settings", () => { + test.use({ viewport: { width: 1100, height: 760 } }); + + test("selects and retains a bundled voice while text to speech is off", async ({ + page, + }) => { + await installMockBridge(page); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await openSettings(page, "voice"); + + const card = page.getByTestId("settings-voice"); + await expect(card).toBeVisible(); + await expect( + page.getByText("Agent text to speech", { exact: true }), + ).toBeVisible(); + + await page.getByTestId("pocket-voice-selector").click(); + await page.getByRole("menuitemradio", { name: "Marius" }).click(); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Marius", + ); + + await page.getByTestId("agent-text-to-speech-toggle").click(); + await expect( + page.getByTestId("agent-text-to-speech-toggle"), + ).toHaveAttribute("aria-checked", "false"); + await expect(page.getByTestId("pocket-voice-controls")).toHaveAttribute( + "aria-disabled", + "true", + ); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Marius", + ); + + const savedCommands = await page.evaluate(() => + (window.__BUZZ_E2E_COMMAND_LOG__ ?? []) + .filter((entry) => + ["set_pocket_voice", "set_tts_enabled"].includes(entry.command), + ) + .map((entry) => ({ command: entry.command, payload: entry.payload })), + ); + expect(savedCommands).toEqual([ + { + command: "set_pocket_voice", + payload: { voiceKey: "pocket:marius" }, + }, + { + command: "set_tts_enabled", + payload: { enabled: false }, + }, + ]); + }); + + test("captures the complete two-voice settings surface", async ({ page }) => { + await installMockBridge(page, { + ttsSettings: { + version: 1, + agentTextToSpeech: true, + voicePreferences: ["pocket:marius"], + }, + }); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await openSettings(page, "voice"); + + const card = page.getByTestId("settings-voice"); + await expect(card).toBeVisible(); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Marius", + ); + await waitForAnimations(page); + await card.screenshot({ path: SCREENSHOT_PATH }); + }); + + test("imports, selects, and safely deletes a local voice", async ({ + page, + }) => { + await installMockBridge(page); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await openSettings(page, "voice"); + + await page.getByTestId("pocket-voice-import").click(); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "My voice", + ); + await expect(page.getByTestId("pocket-voice-delete")).toBeVisible(); + await page.getByRole("button", { name: "Preview" }).click(); + + await page.getByTestId("pocket-voice-delete").click(); + await expect(page.getByText("Delete imported voice?")).toBeVisible(); + await page.getByTestId("confirm-pocket-voice-delete").click(); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Mary", + ); + await expect(page.getByTestId("pocket-voice-delete")).toBeHidden(); + await page.getByRole("button", { name: "Preview" }).click(); + + const mutations = await page.evaluate(() => + (window.__BUZZ_E2E_COMMAND_LOG__ ?? []) + .filter((entry) => + [ + "import_pocket_voice", + "preview_pocket_voice", + "delete_pocket_voice", + ].includes(entry.command), + ) + .map((entry) => ({ command: entry.command, payload: entry.payload })), + ); + expect(mutations).toEqual([ + { command: "import_pocket_voice", payload: {} }, + { + command: "preview_pocket_voice", + payload: { voiceKey: `pocket:imported:${"1".repeat(64)}` }, + }, + { + command: "delete_pocket_voice", + payload: { voiceKey: `pocket:imported:${"1".repeat(64)}` }, + }, + { + command: "preview_pocket_voice", + payload: { voiceKey: "pocket:mary" }, + }, + ]); + }); + + test("keeps the selected voice unchanged when the native picker is cancelled", async ({ + page, + }) => { + await installMockBridge(page, { pocketVoiceImportResult: "cancel" }); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await openSettings(page, "voice"); + + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Mary", + ); + await page.getByTestId("pocket-voice-import").click(); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Mary", + ); + await expect(page.getByTestId("pocket-voice-delete")).toBeHidden(); + await expect(page.getByTestId("voice-settings-error")).toBeHidden(); + + const audioCommands = await page.evaluate(() => + (window.__BUZZ_E2E_COMMAND_LOG__ ?? []).filter((entry) => + ["preview_pocket_voice", "delete_pocket_voice"].includes(entry.command), + ), + ); + expect(audioCommands).toEqual([]); + }); + + test("surfaces invalid or unsupported WAV errors without changing selection", async ({ + page, + }) => { + await installMockBridge(page, { pocketVoiceImportResult: "invalid" }); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await openSettings(page, "voice"); + + await page.getByTestId("pocket-voice-import").click(); + await expect(page.getByTestId("voice-settings-error")).toContainText( + "Voice WAV must contain PCM or 32-bit float audio", + ); + await expect(page.getByTestId("pocket-voice-selector")).toContainText( + "Mary", + ); + await expect(page.getByTestId("pocket-voice-delete")).toBeHidden(); + }); +}); diff --git a/desktop/tests/helpers/bridge.ts b/desktop/tests/helpers/bridge.ts index d65a260f30..00ed9f5839 100644 --- a/desktop/tests/helpers/bridge.ts +++ b/desktop/tests/helpers/bridge.ts @@ -128,6 +128,13 @@ export type MockAgentMemoryListing = { }; type MockBridgeOptions = { + ttsSettings?: { + version: number; + agentTextToSpeech: boolean; + voicePreferences: string[]; + }; + /** Native picker boundary result for Pocket voice import tests. */ + pocketVoiceImportResult?: "success" | "cancel" | "invalid"; /** Advertised HEAD for the first mock project without adding that branch. */ projectHeadBranch?: string; /** Relay NIP-11 identity used to sign authoritative repository state. */ diff --git a/desktop/tests/helpers/settings.ts b/desktop/tests/helpers/settings.ts index a63b52453e..c26c0e9195 100644 --- a/desktop/tests/helpers/settings.ts +++ b/desktop/tests/helpers/settings.ts @@ -3,6 +3,7 @@ import { expect, type Page } from "@playwright/test"; type SettingsSection = | "profile" | "notifications" + | "voice" | "agents" | "channel-templates" | "compute" diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 4dd1142801..ae26fc6ba8 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -84,6 +84,9 @@ run_unit_tests() { run_test_step "buzz-auth unit tests" \ cargo test -p buzz-auth --lib -- --nocapture + run_test_step "buzz-voice tests" \ + cargo test -p buzz-voice --lib -- --nocapture + # buzz-db migrator/lint unit tests (no infra): guard the embedded-migrator # invariant (exactly the consolidated 0001; cutover/backfill stays an operator # script, not startup state) and the tenant-scoping lints. The Postgres-backed