diff --git a/.gitignore b/.gitignore index 65ddcaf1c4..3ef0226174 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /target/ /dist/ /admin-web/dist/ +/mobile/.generated/voice/ # lefthook-generated hook scripts (machine-specific) .hooks/ diff --git a/Cargo.lock b/Cargo.lock index d3ca5d916e..5faa667905 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1295,6 +1295,7 @@ dependencies = [ "ort", "ort-sys", "rand 0.10.1", + "regex", "sentencepiece-model", "serde", "serde_json", @@ -1302,6 +1303,14 @@ dependencies = [ "tokenizers", ] +[[package]] +name = "buzz-voice-mobile" +version = "0.1.0" +dependencies = [ + "buzz-voice", + "serde_json", +] + [[package]] name = "buzz-workflow" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f7bbad294b..c916e09040 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ members = [ "crates/buzz-relay-mesh", "crates/buzz-dev-mcp", "crates/buzz-voice", + "crates/buzz-voice-mobile", "examples/countdown-bot", ] exclude = ["desktop/src-tauri", "crates/sherpa-onnx-sys"] diff --git a/crates/buzz-voice-mobile/Cargo.toml b/crates/buzz-voice-mobile/Cargo.toml new file mode 100644 index 0000000000..bd3ba760f1 --- /dev/null +++ b/crates/buzz-voice-mobile/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "buzz-voice-mobile" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Minimal mobile C ABI for Buzz Pocket TTS" + +[lib] +crate-type = ["rlib", "staticlib", "cdylib"] + +[features] +default = ["static"] +static = ["buzz-voice/static"] +shared = ["buzz-voice/shared"] + +[dependencies] +buzz-voice = { path = "../buzz-voice", default-features = false } +serde_json = { workspace = true } diff --git a/crates/buzz-voice-mobile/include/buzz_voice_mobile.h b/crates/buzz-voice-mobile/include/buzz_voice_mobile.h new file mode 100644 index 0000000000..a82fac4d21 --- /dev/null +++ b/crates/buzz-voice-mobile/include/buzz_voice_mobile.h @@ -0,0 +1,32 @@ +#ifndef BUZZ_VOICE_MOBILE_H +#define BUZZ_VOICE_MOBILE_H + +#include +#include + +typedef struct BuzzVoiceEngineResult { + void *engine; + char *error; +} BuzzVoiceEngineResult; + +typedef struct BuzzVoicePcm { + int16_t *samples; + size_t len; + uint32_t sample_rate; + char *error; +} BuzzVoicePcm; + +#define BUZZ_VOICE_PRECISION_FP32 0 +#define BUZZ_VOICE_PRECISION_INT8 1 + +BuzzVoiceEngineResult buzz_voice_engine_create(const char *model_dir, + uint8_t precision); +char *buzz_voice_prepare_chunks_json(void *engine, const char *text); +BuzzVoicePcm buzz_voice_engine_synthesize(void *engine, const char *text); +void buzz_voice_engine_cancel(void *engine); +void buzz_voice_engine_reset_cancel(void *engine); +void buzz_voice_engine_destroy(void *engine); +void buzz_voice_pcm_free(BuzzVoicePcm pcm); +void buzz_voice_string_free(char *value); + +#endif diff --git a/crates/buzz-voice-mobile/src/lib.rs b/crates/buzz-voice-mobile/src/lib.rs new file mode 100644 index 0000000000..ca6339ad28 --- /dev/null +++ b/crates/buzz-voice-mobile/src/lib.rs @@ -0,0 +1,342 @@ +//! Minimal C ABI for running [`buzz_voice::pocket`] on mobile. + +#![deny(unsafe_op_in_unsafe_fn)] + +use std::ffi::{c_char, c_void, CStr, CString}; +use std::ptr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use buzz_voice::pocket::{ + load_text_to_speech_with_options, load_voice_style, PocketLoadOptions, PocketModelSelection, + PocketPrecision, PocketTts, VoiceStyle, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT, +}; +use buzz_voice::preparation::{prepare_tts_chunks, shape_tts_chunk}; + +struct Engine { + tts: PocketTts, + voice: VoiceStyle, + cancelled: Arc, +} + +/// Result returned by [`buzz_voice_engine_create`]. +#[repr(C)] +pub struct BuzzVoiceEngineResult { + /// Opaque engine handle, or null when creation failed. + pub engine: *mut c_void, + /// Owned UTF-8 error string, or null on success. + pub error: *mut c_char, +} + +/// Owned 24 kHz mono signed-16-bit PCM returned by +/// [`buzz_voice_engine_synthesize`]. +#[repr(C)] +pub struct BuzzVoicePcm { + /// Owned sample buffer, or null when synthesis failed. + pub samples: *mut i16, + /// Number of samples in `samples`. + pub len: usize, + /// Sample rate in Hz. + pub sample_rate: u32, + /// Owned UTF-8 error string, or null on success. + pub error: *mut c_char, +} + +fn owned_error(message: impl Into) -> *mut c_char { + let message = message.into().replace('\0', " "); + CString::new(message) + .map(CString::into_raw) + .unwrap_or(ptr::null_mut()) +} + +fn engine_error(message: impl Into) -> BuzzVoiceEngineResult { + BuzzVoiceEngineResult { + engine: ptr::null_mut(), + error: owned_error(message), + } +} + +fn pcm_error(message: impl Into) -> BuzzVoicePcm { + BuzzVoicePcm { + samples: ptr::null_mut(), + len: 0, + sample_rate: SAMPLE_RATE, + error: owned_error(message), + } +} + +fn input_string(value: *const c_char, label: &str) -> Result { + if value.is_null() { + return Err(format!("{label} is null")); + } + // SAFETY: The C ABI requires a non-null, NUL-terminated string that stays + // alive for the duration of this call. We copy it before returning. + let value = unsafe { CStr::from_ptr(value) }; + value + .to_str() + .map(str::to_owned) + .map_err(|error| format!("{label} is not UTF-8: {error}")) +} + +/// Create and retain one Pocket engine and its reference voice. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_create( + model_dir: *const c_char, + precision: u8, +) -> BuzzVoiceEngineResult { + let model_dir = match input_string(model_dir, "model_dir") { + Ok(value) => value, + Err(error) => return engine_error(error), + }; + let precision = match precision { + value if value == PocketPrecision::Fp32 as u8 => PocketPrecision::Fp32, + value if value == PocketPrecision::Int8 as u8 => PocketPrecision::Int8, + value => return engine_error(format!("unsupported Pocket precision: {value}")), + }; + let voice_path = std::path::Path::new(&model_dir) + .join(DEFAULT_VOICE) + .with_extension(VOICE_FILE_EXT); + let options = PocketLoadOptions { + model: PocketModelSelection::English2026_04(precision), + ..PocketLoadOptions::default() + }; + let tts = match load_text_to_speech_with_options(&model_dir, options) { + Ok(value) => value, + Err(error) => return engine_error(error), + }; + let voice = match load_voice_style(&voice_path) { + Ok(value) => value, + Err(error) => return engine_error(error), + }; + // Pay the model's first-generation setup cost during engine creation so + // the first submitted assistant sentence is not the warm-up utterance. + if let Err(error) = tts.synth_chunk("warmup", "en", &voice, 1) { + eprintln!("buzz-voice-mobile: Pocket warm-up failed: {error}"); + } + let engine = Box::new(Engine { + tts, + voice, + cancelled: Arc::new(AtomicBool::new(false)), + }); + BuzzVoiceEngineResult { + engine: Box::into_raw(engine).cast(), + error: ptr::null_mut(), + } +} + +/// Synthesize one utterance into owned signed-16-bit PCM. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_synthesize( + engine: *mut c_void, + text: *const c_char, +) -> BuzzVoicePcm { + if engine.is_null() { + return pcm_error("engine is null"); + } + let text = match input_string(text, "text") { + Ok(value) => value, + Err(error) => return pcm_error(error), + }; + // SAFETY: A handle returned by `buzz_voice_engine_create` remains owned by + // the caller until `buzz_voice_engine_destroy`; synthesis and destroy must + // not overlap. Cancellation only touches the atomic flag. + let engine = unsafe { &*(engine.cast::()) }; + let cancelled = Arc::clone(&engine.cancelled); + let samples = match engine.tts.synth_chunk_with_callback( + &text, + "en", + &engine.voice, + 1, + Some(move |_: &[f32], _| !cancelled.load(Ordering::Acquire)), + ) { + Ok(value) => value, + Err(error) => return pcm_error(error), + }; + if engine.cancelled.load(Ordering::Acquire) { + return pcm_error("synthesis cancelled"); + } + + let samples: Box<[i16]> = shape_tts_chunk(samples) + .into_iter() + .map(|sample| { + let sample = if sample.is_finite() { sample } else { 0.0 }; + (sample.clamp(-1.0, 1.0) * i16::MAX as f32).round() as i16 + }) + .collect::>() + .into_boxed_slice(); + let len = samples.len(); + BuzzVoicePcm { + samples: Box::into_raw(samples).cast::(), + len, + sample_rate: SAMPLE_RATE, + error: ptr::null_mut(), + } +} + +/// Prepare submitted assistant text and return an owned JSON string array. +/// +/// The returned pointer is released with [`buzz_voice_string_free`]. A null +/// pointer indicates invalid input or an unexpected serialization failure. +#[no_mangle] +pub extern "C" fn buzz_voice_prepare_chunks_json( + engine: *mut c_void, + text: *const c_char, +) -> *mut c_char { + if engine.is_null() { + return ptr::null_mut(); + } + let text = match input_string(text, "text") { + Ok(value) => value, + Err(_) => return ptr::null_mut(), + }; + // SAFETY: See `buzz_voice_engine_synthesize`. + let engine = unsafe { &*(engine.cast::()) }; + let mut chunks = Vec::new(); + for prepared in prepare_tts_chunks(&text) { + let model_chunks = match engine.tts.split_text_into_chunks(&prepared) { + Ok(value) => value, + Err(_) => return ptr::null_mut(), + }; + chunks.extend(model_chunks); + } + let json = match serde_json::to_string(&chunks) { + Ok(value) => value, + Err(_) => return ptr::null_mut(), + }; + CString::new(json) + .map(CString::into_raw) + .unwrap_or(ptr::null_mut()) +} + +/// Request cancellation of the current and subsequent synthesis calls. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_cancel(engine: *mut c_void) { + if engine.is_null() { + return; + } + // SAFETY: See `buzz_voice_engine_synthesize`. The engine remains alive + // while this atomic cancellation request is issued. + let engine = unsafe { &*(engine.cast::()) }; + engine.cancelled.store(true, Ordering::Release); +} + +/// Clear sticky cancellation before beginning a new playback request. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_reset_cancel(engine: *mut c_void) { + if engine.is_null() { + return; + } + // SAFETY: See `buzz_voice_engine_synthesize`. + let engine = unsafe { &*(engine.cast::()) }; + engine.cancelled.store(false, Ordering::Release); +} + +/// Destroy an engine after any synthesis call has returned. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_destroy(engine: *mut c_void) { + if engine.is_null() { + return; + } + // SAFETY: The pointer came from `Box::into_raw` in create and is consumed + // exactly once by this function. + unsafe { drop(Box::from_raw(engine.cast::())) }; +} + +/// Release a PCM value returned by [`buzz_voice_engine_synthesize`]. +#[no_mangle] +pub extern "C" fn buzz_voice_pcm_free(pcm: BuzzVoicePcm) { + if !pcm.samples.is_null() { + // SAFETY: The allocation came from `Box<[i16]>` in synthesis and is + // consumed exactly once here with the same slice length. + let slice = ptr::slice_from_raw_parts_mut(pcm.samples, pcm.len); + unsafe { drop(Box::from_raw(slice)) }; + } + buzz_voice_string_free(pcm.error); +} + +/// Release an error string returned by this ABI. +#[no_mangle] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub extern "C" fn buzz_voice_string_free(value: *mut c_char) { + if value.is_null() { + return; + } + // SAFETY: Error pointers are produced by `CString::into_raw` and consumed + // exactly once here. + unsafe { drop(CString::from_raw(value)) }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn null_inputs_return_owned_errors() { + let engine = buzz_voice_engine_create(ptr::null(), PocketPrecision::Fp32 as u8); + assert!(engine.engine.is_null()); + assert!(!engine.error.is_null()); + buzz_voice_string_free(engine.error); + + let pcm = buzz_voice_engine_synthesize(ptr::null_mut(), ptr::null()); + assert!(pcm.samples.is_null()); + assert_eq!(pcm.len, 0); + assert_eq!(pcm.sample_rate, SAMPLE_RATE); + assert!(!pcm.error.is_null()); + buzz_voice_pcm_free(pcm); + } + + #[test] + fn chunk_preparation_requires_an_engine() { + let input = CString::new("First. Second. Third.").expect("test input"); + let json = buzz_voice_prepare_chunks_json(ptr::null_mut(), input.as_ptr()); + assert!(json.is_null()); + } + + #[test] + fn invalid_precision_returns_an_owned_error() { + let model = CString::new("/tmp/pocket").expect("test path"); + let engine = buzz_voice_engine_create(model.as_ptr(), u8::MAX); + assert!(engine.engine.is_null()); + assert!(!engine.error.is_null()); + buzz_voice_string_free(engine.error); + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"] + fn april_fp32_abi_prepares_and_synthesizes_valid_pcm() { + let model = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to an April FP32 bundle"); + let model = CString::new(model).expect("test model path"); + let result = buzz_voice_engine_create(model.as_ptr(), PocketPrecision::Fp32 as u8); + if !result.error.is_null() { + // SAFETY: The ABI returned a live NUL-terminated owned error. + let error = unsafe { CStr::from_ptr(result.error) } + .to_string_lossy() + .into_owned(); + buzz_voice_string_free(result.error); + panic!("engine creation failed: {error}"); + } + assert!(!result.engine.is_null()); + + let text = CString::new( + "Pocket voice is running on mobile. This second sentence verifies shared chunking.", + ) + .expect("test text"); + let json = buzz_voice_prepare_chunks_json(result.engine, text.as_ptr()); + assert!(!json.is_null()); + // SAFETY: The ABI returned a live NUL-terminated owned JSON string. + let chunks: Vec = + serde_json::from_slice(unsafe { CStr::from_ptr(json) }.to_bytes()).expect("chunk JSON"); + buzz_voice_string_free(json); + assert_eq!(chunks.len(), 2); + + let prompt = CString::new(chunks[0].as_str()).expect("prepared prompt"); + let pcm = buzz_voice_engine_synthesize(result.engine, prompt.as_ptr()); + assert!(pcm.error.is_null()); + assert!(!pcm.samples.is_null()); + assert!(pcm.len > 24_000 / 4); + assert_eq!(pcm.sample_rate, 24_000); + buzz_voice_pcm_free(pcm); + buzz_voice_engine_destroy(result.engine); + } +} diff --git a/crates/buzz-voice/Cargo.toml b/crates/buzz-voice/Cargo.toml index 6576fba57b..e492a9e7f1 100644 --- a/crates/buzz-voice/Cargo.toml +++ b/crates/buzz-voice/Cargo.toml @@ -16,6 +16,7 @@ shared = ["sherpa-onnx/shared"] ort = { version = "=2.0.0-rc.12", default-features = false, features = ["api-24", "ndarray", "std"] } ort-sys = { version = "=2.0.0-rc.12", features = ["disable-linking"] } rand = { workspace = true } +regex = "1" sentencepiece-model = "0.1" serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/buzz-voice/src/lib.rs b/crates/buzz-voice/src/lib.rs index e2ce3f255b..6d46e03de1 100644 --- a/crates/buzz-voice/src/lib.rs +++ b/crates/buzz-voice/src/lib.rs @@ -1,3 +1,4 @@ //! Reusable local voice primitives for Buzz. pub mod pocket; +pub mod preparation; diff --git a/crates/buzz-voice/src/preparation.rs b/crates/buzz-voice/src/preparation.rs new file mode 100644 index 0000000000..5661a1aad3 --- /dev/null +++ b/crates/buzz-voice/src/preparation.rs @@ -0,0 +1,784 @@ +//! Shared text preparation and PCM shaping for TTS output. +//! +//! Mental model: +//! +//! ```text +//! raw agent text +//! → strip fenced code blocks → "code block omitted" +//! → strip inline code → bare text +//! → strip URLs → "link omitted" +//! → strip markdown markers → plain text +//! → strip emoji → (removed) +//! → numbers → words → "forty two" +//! → collapse whitespace → clean string +//! ``` +//! +//! Also provides `split_sentences` — the single sentence-boundary splitter used +//! by both the TTS batching pipeline and the Supertonic text chunker. + +use regex::Regex; +use std::sync::LazyLock; + +// ── Sentence splitting ──────────────────────────────────────────────────────── + +/// Regex: a sentence-ending punctuation mark followed by whitespace. +static RE_SENTENCE_BOUNDARY: LazyLock = LazyLock::new(|| Regex::new(r"([.!?])\s+").unwrap()); + +/// Common abbreviations that end with a period but are NOT sentence boundaries. +const ABBREVIATIONS: &[&str] = &[ + "Dr.", "Mr.", "Mrs.", "Ms.", "Prof.", "Sr.", "Jr.", "St.", "Ave.", "Rd.", "Blvd.", "Dept.", + "Inc.", "Ltd.", "Co.", "Corp.", "etc.", "vs.", "i.e.", "e.g.", "Ph.D.", +]; + +/// Split text into sentence-sized chunks. +/// +/// Combines regex-based boundary detection with: +/// - Abbreviation awareness (`Dr.`, `Mr.`, etc. don't split) +/// - Digit-before-period check (avoids splitting `1.` `2.` numbered lists) +/// - `\n` and `—` treated as sentence breaks +/// +/// Returns non-empty, trimmed strings. +pub fn split_sentences(text: &str) -> Vec { + // First, split on newlines and em-dashes to get coarse segments. + let coarse: Vec<&str> = text.split(['\n', '—']).collect(); + + let mut sentences = Vec::new(); + + for segment in coarse { + let segment = segment.trim(); + if segment.is_empty() { + continue; + } + // Within each segment, split on sentence-ending punctuation. + let matches: Vec<_> = RE_SENTENCE_BOUNDARY.find_iter(segment).collect(); + if matches.is_empty() { + sentences.push(segment.to_string()); + continue; + } + + let mut last_end = 0usize; + for m in &matches { + let before = &segment[last_end..m.start()]; + let punc_char = &segment[m.start()..m.start() + 1]; + + // Skip if this looks like an abbreviation. + let combined = format!("{}{}", before.trim(), punc_char); + let is_abbrev = ABBREVIATIONS.iter().any(|a| combined.ends_with(a)); + + // Skip if the character before the period is a digit (numbered list). + let is_digit_period = punc_char == "." + && !before.is_empty() + && before.ends_with(|c: char| c.is_ascii_digit()); + + if !is_abbrev && !is_digit_period { + let piece = segment[last_end..m.end()].trim(); + if !piece.is_empty() { + sentences.push(piece.to_string()); + } + last_end = m.end(); + } + } + + if last_end < segment.len() { + let tail = segment[last_end..].trim(); + if !tail.is_empty() { + sentences.push(tail.to_string()); + } + } + } + + if sentences.is_empty() { + vec![text.to_string()] + } else { + sentences + } +} + +// ── Public API ──────────────────────────────────────────────────────────────── + +/// Prepare `text` for TTS synthesis. +/// +/// Applies in order: +/// 1. Fenced code blocks → "code block omitted" +/// 2. Inline code → bare content (backticks stripped) +/// 3. URLs → "link omitted" +/// 4. Markdown bold/italic/underline markers stripped +/// 5. Emoji stripped +/// 6. Numbers → words (integers 0–999, times HH:MM) +/// 7. Excess whitespace collapsed +pub fn preprocess_for_tts(text: &str) -> String { + let s = strip_fenced_code_blocks(text); + let s = strip_inline_code(&s); + let s = strip_urls(&s); + let s = strip_markdown_markers(&s); + let s = strip_emoji(&s); + let s = expand_numbers(&s); + let s = collapse_whitespace(&s); + // Filter trivially short results — ".", ",", etc. would be spoken as + // "period", "comma" by TTS. Agents that have nothing relevant to say + // should not respond at all, but defense-in-depth catches edge cases. + if s.len() <= 1 { + return String::new(); + } + s +} + +// ── Step implementations ────────────────────────────────────────────────────── + +/// Replace fenced code blocks with "code block omitted". +/// +/// Handles both ` ``` ` and `~~~` fences. Multi-line aware. +fn strip_fenced_code_blocks(text: &str) -> String { + let s = replace_fenced(text, "```"); + replace_fenced(&s, "~~~") +} + +fn replace_fenced(text: &str, fence: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + match rest.find(fence) { + None => { + out.push_str(rest); + break; + } + Some(start) => { + // Everything before the opening fence. + out.push_str(&rest[..start]); + rest = &rest[start + fence.len()..]; + // Skip optional language tag on the same line. + if let Some(nl) = rest.find('\n') { + rest = &rest[nl + 1..]; + } + // Find the closing fence. + match rest.find(fence) { + None => { + // Unclosed fence — treat rest as omitted. + out.push_str(" code block omitted "); + break; + } + Some(end) => { + out.push_str(" code block omitted "); + rest = &rest[end + fence.len()..]; + // Skip trailing newline after closing fence. + if rest.starts_with('\n') { + rest = &rest[1..]; + } + } + } + } + } + } + out +} + +/// Strip backtick-delimited inline code, leaving the inner text. +/// +/// Single-backtick only — triple backtick already handled above. +fn strip_inline_code(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + match rest.find('`') { + None => { + out.push_str(rest); + break; + } + Some(start) => { + out.push_str(&rest[..start]); + rest = &rest[start + 1..]; + match rest.find('`') { + None => { + // Unclosed — emit as-is. + out.push_str(rest); + break; + } + Some(end) => { + out.push_str(&rest[..end]); + rest = &rest[end + 1..]; + } + } + } + } + } + out +} + +/// Replace http/https URLs with "link omitted". +/// +/// Trailing sentence-ending punctuation (`.`, `!`, `?`) that immediately follows +/// a URL and is at end-of-string or followed by whitespace is preserved so that +/// sentence splitting and TTS prosody are not degraded. +/// +/// Example: `"See https://x.y/z."` → `"See link omitted."` +fn strip_urls(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + // Find the earliest URL prefix. + let http = rest.find("http://"); + let https = rest.find("https://"); + let url_start = match (http, https) { + (None, None) => { + out.push_str(rest); + break; + } + (Some(a), None) => a, + (None, Some(b)) => b, + (Some(a), Some(b)) => a.min(b), + }; + out.push_str(&rest[..url_start]); + rest = &rest[url_start..]; + // Consume until whitespace or structural delimiter. + let url_end = rest + .find(|c: char| c.is_whitespace() || c == ')' || c == ']' || c == '"' || c == '\'') + .unwrap_or(rest.len()); + let url_token = &rest[..url_end]; + rest = &rest[url_end..]; + + // Check if the URL token ends with sentence-ending punctuation that + // belongs to the surrounding sentence rather than the URL itself. + // A trailing `.`, `!`, or `?` is preserved when it is at end-of-string + // or followed by whitespace (i.e. it is a sentence boundary). + let trailing_punct = if url_token.ends_with(['.', '!', '?']) { + let after = rest; // rest is already past url_end + if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { + // Preserve the trailing punctuation. + &url_token[url_token.len() - 1..] + } else { + "" + } + } else { + "" + }; + + out.push_str("link omitted"); + out.push_str(trailing_punct); + } + out +} + +/// Strip `**`, `*`, `__`, `_emphasis_`, `~~` markdown markers. +/// +/// Underscores are only stripped when they wrap a word (`_text_`). +/// Standalone underscores (e.g. `snake_case` identifiers) are preserved. +fn strip_markdown_markers(text: &str) -> String { + // Order matters: strip multi-char markers before single-char. + let s = text.replace("**", ""); + let s = s.replace("__", ""); + let s = s.replace("~~", ""); + let s = s.replace('*', ""); + strip_underscore_emphasis(&s) +} + +/// Strip `_text_` emphasis markers while preserving underscores in identifiers. +/// +/// A `_` is treated as an emphasis delimiter only when it is preceded by +/// whitespace or the start of the string AND followed by a non-whitespace char, +/// or vice-versa for the closing delimiter. +fn strip_underscore_emphasis(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let chars: Vec = text.chars().collect(); + let len = chars.len(); + let mut i = 0; + while i < len { + if chars[i] == '_' { + // Opening delimiter: preceded by whitespace/start, followed by non-whitespace. + let prev_is_boundary = i == 0 || chars[i - 1].is_whitespace(); + let next_is_nonspace = i + 1 < len && !chars[i + 1].is_whitespace(); + if prev_is_boundary && next_is_nonspace { + // Look for a matching closing `_`. + if let Some(close) = (i + 1..len).find(|&j| { + chars[j] == '_' + && !chars[j - 1].is_whitespace() + && (j + 1 >= len + || chars[j + 1].is_whitespace() + || chars[j + 1].is_ascii_punctuation()) + }) { + // Emit the inner text without the delimiters. + for &ch in &chars[i + 1..close] { + out.push(ch); + } + i = close + 1; + continue; + } + } + // Not an emphasis delimiter — emit as-is. + out.push('_'); + } else { + out.push(chars[i]); + } + i += 1; + } + out +} + +/// Strip Unicode emoji (characters in common emoji ranges). +/// +/// Covers the main Emoji block (U+1F300–U+1FAFF) and supplemental ranges. +/// ASCII emoticons like `:)` are left as-is. +fn strip_emoji(text: &str) -> String { + text.chars().filter(|&c| !is_emoji(c)).collect() +} + +#[inline] +fn is_emoji(c: char) -> bool { + matches!(c, + '\u{1F300}'..='\u{1FAFF}' // Misc symbols, emoticons, transport, etc. + | '\u{2600}'..='\u{27BF}' // Misc symbols, dingbats + | '\u{FE00}'..='\u{FE0F}' // Variation selectors + | '\u{1F000}'..='\u{1F02F}'// Mahjong/domino tiles + | '\u{1F0A0}'..='\u{1F0FF}'// Playing cards + | '\u{200D}' // Zero-width joiner (used in emoji sequences) + | '\u{20E3}' // Combining enclosing keycap + ) +} + +/// Expand numbers to spoken words. +/// +/// Handles: +/// - Times: `HH:MM` → "eleven thirty" +/// - Integers 0–999,999 +/// - Leaves other numeric strings (e.g. "3.14", "1000000+") as-is. +fn expand_numbers(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.char_indices().peekable(); + + while let Some((i, c)) = chars.next() { + if c.is_ascii_digit() { + // Collect the full token (digits, colon, dots). + let start = i; + let mut end = i + c.len_utf8(); + while let Some(&(j, nc)) = chars.peek() { + if nc.is_ascii_digit() || nc == ':' || nc == '.' { + end = j + nc.len_utf8(); + chars.next(); + } else { + break; + } + } + let token = &text[start..end]; + out.push_str(&expand_numeric_token(token)); + } else { + out.push(c); + } + } + out +} + +fn expand_numeric_token(token: &str) -> String { + // Strip trailing punctuation that the token collector may have included + // (e.g. "11:30." from "at 11:30.") before attempting to parse. + let token = token.trim_end_matches(|c: char| !c.is_ascii_digit()); + + // Time: HH:MM + if let Some(colon) = token.find(':') { + let h = &token[..colon]; + let m = &token[colon + 1..]; + if let (Ok(hh), Ok(mm)) = (h.parse::(), m.parse::()) { + if hh < 24 && mm < 60 { + let hour_word = int_to_words(hh); + let min_word = if mm == 0 { + String::new() + } else if mm < 10 { + // "9:05" → "nine oh five" (not "nine five") + format!(" oh {}", int_to_words(mm)) + } else { + format!(" {}", int_to_words(mm)) + }; + return format!("{}{}", hour_word, min_word); + } + } + // Not a valid time — return as-is. + return token.to_string(); + } + + // Plain integer 0–999,999. + if token.chars().all(|c| c.is_ascii_digit()) { + if let Ok(n) = token.parse::() { + if n <= 999_999 { + return int_to_words(n); + } + } + } + + // Anything else (decimals, millions+) — leave as-is. + token.to_string() +} + +/// Convert an integer 0–999,999 to English words. +fn int_to_words(n: u32) -> String { + const ONES: &[&str] = &[ + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", + "ten", + "eleven", + "twelve", + "thirteen", + "fourteen", + "fifteen", + "sixteen", + "seventeen", + "eighteen", + "nineteen", + ]; + const TENS: &[&str] = &[ + "", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety", + ]; + + if n < 20 { + return ONES[n as usize].to_string(); + } + if n < 100 { + let ten = TENS[(n / 10) as usize]; + let one = n % 10; + return if one == 0 { + ten.to_string() + } else { + format!("{} {}", ten, ONES[one as usize]) + }; + } + if n < 1000 { + let hundreds = n / 100; + let remainder = n % 100; + let hundred_word = format!("{} hundred", ONES[hundreds as usize]); + return if remainder == 0 { + hundred_word + } else { + format!("{} {}", hundred_word, int_to_words(remainder)) + }; + } + // 1,000–999,999 + let thousands = n / 1000; + let remainder = n % 1000; + let thousand_word = format!("{} thousand", int_to_words(thousands)); + if remainder == 0 { + thousand_word + } else { + format!("{} {}", thousand_word, int_to_words(remainder)) + } +} + +/// Collapse runs of whitespace (spaces, tabs, newlines) to a single space. +/// Trims leading/trailing whitespace. +fn collapse_whitespace(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut prev_space = true; // Start true to trim leading whitespace. + for c in text.chars() { + if c.is_whitespace() { + if !prev_space { + out.push(' '); + prev_space = true; + } + } else { + out.push(c); + prev_space = false; + } + } + // Trim trailing space. + if out.ends_with(' ') { + out.pop(); + } + out +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +/// Character budget for greedy synthesis chunks after the latency-critical +/// first sentence. +pub const TTS_CHUNK_MAX_CHARS: usize = 200; + +/// Pocket's output sample rate. +pub const TTS_SAMPLE_RATE: usize = 24_000; +const FADE_OUT_SAMPLES: usize = TTS_SAMPLE_RATE * 8 / 1_000; +const LEAD_IN_SAMPLES: usize = TTS_SAMPLE_RATE * 20 / 1_000; +const TRAILING_SILENCE_SAMPLES: usize = TTS_SAMPLE_RATE * 80 / 1_000; + +/// Convert submitted assistant text into desktop-compatible synthesis chunks. +/// +/// The first sentence stands alone to minimize time to first playback. +/// Remaining sentences are packed greedily up to 200 characters. +pub fn prepare_tts_chunks(raw: &str) -> Vec { + let text = preprocess_for_tts(raw); + if text.is_empty() { + return Vec::new(); + } + let sentences: Vec = split_sentences(&text) + .into_iter() + .filter(|sentence| !sentence.trim().is_empty()) + .collect(); + group_sentences_into_chunks(&sentences, TTS_CHUNK_MAX_CHARS) +} + +/// Group already-prepared sentences using Pocket's desktop chunking policy. +pub fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec { + let mut chunks: Vec = Vec::new(); + for (index, sentence) in sentences.iter().enumerate() { + let sentence = sentence.trim(); + if sentence.is_empty() { + continue; + } + if index == 0 || chunks.is_empty() { + chunks.push(sentence.to_owned()); + continue; + } + let can_merge = chunks.len() > 1 + && chunks + .last() + .is_some_and(|chunk| chunk.len() + 1 + sentence.len() <= max_chars); + if can_merge { + if let Some(chunk) = chunks.last_mut() { + chunk.push(' '); + chunk.push_str(sentence); + } + } else { + chunks.push(sentence.to_owned()); + } + } + chunks +} + +/// Clamp and shape a Pocket chunk for glitch-free playback. +/// +/// Applies the desktop 8 ms fade-out, then prepends 20 ms and appends 80 ms +/// of silence, preserving the established 100 ms inter-chunk gap. +pub fn shape_tts_chunk(samples: Vec) -> Vec { + let mut audio: Vec = samples + .into_iter() + .map(|sample| sample.clamp(-1.0, 1.0)) + .collect(); + let fade = FADE_OUT_SAMPLES.min(audio.len() / 2); + for index in 0..fade { + let sample_index = audio.len() - 1 - index; + audio[sample_index] *= index as f32 / fade as f32; + } + + let mut shaped = Vec::with_capacity(LEAD_IN_SAMPLES + audio.len() + TRAILING_SILENCE_SAMPLES); + shaped.resize(LEAD_IN_SAMPLES, 0.0); + shaped.extend(audio); + shaped.resize(shaped.len() + TRAILING_SILENCE_SAMPLES, 0.0); + shaped +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_fenced_code_block() { + let input = "Here is some code:\n```rust\nfn main() {}\n```\nDone."; + let out = preprocess_for_tts(input); + assert!(out.contains("code block omitted"), "got: {out}"); + assert!(!out.contains("fn main"), "got: {out}"); + } + + #[test] + fn strips_inline_code() { + let out = preprocess_for_tts("Call `foo()` now."); + assert_eq!(out, "Call foo() now."); + } + + #[test] + fn strips_urls() { + let out = preprocess_for_tts("See https://example.com for details."); + assert!(out.contains("link omitted"), "got: {out}"); + assert!(!out.contains("example.com"), "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_period() { + // Trailing `.` at end of sentence must be preserved for sentence splitting. + let out = strip_urls("See https://x.y/z."); + assert_eq!(out, "See link omitted.", "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_exclamation() { + let out = strip_urls("Visit https://example.com!"); + assert_eq!(out, "Visit link omitted!", "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_question() { + let out = strip_urls("Did you see https://example.com?"); + assert_eq!(out, "Did you see link omitted?", "got: {out}"); + } + + #[test] + fn strips_url_mid_sentence_no_punct_preserved() { + // URL in the middle of a sentence — no trailing punct to preserve. + let out = strip_urls("Check https://example.com for more info."); + assert_eq!(out, "Check link omitted for more info.", "got: {out}"); + } + + #[test] + fn strips_bold_italic() { + let out = preprocess_for_tts("**bold** and *italic* and _under_"); + assert_eq!(out, "bold and italic and under"); + } + + #[test] + fn preserves_standalone_underscores() { + // snake_case identifiers should not be mangled. + let out = preprocess_for_tts("call foo_bar() or baz_qux"); + assert!(out.contains("foo_bar"), "got: {out}"); + assert!(out.contains("baz_qux"), "got: {out}"); + } + + #[test] + fn strips_tilde_fenced_block() { + let input = "Here:\n~~~python\nprint('hi')\n~~~\nDone."; + let out = preprocess_for_tts(input); + assert!(out.contains("code block omitted"), "got: {out}"); + assert!(!out.contains("print"), "got: {out}"); + } + + #[test] + fn expands_integers() { + assert_eq!(preprocess_for_tts("42"), "forty two"); + assert_eq!(preprocess_for_tts("0"), "zero"); + assert_eq!(preprocess_for_tts("11"), "eleven"); + assert_eq!(preprocess_for_tts("100"), "one hundred"); + } + + #[test] + fn expands_thousands() { + assert_eq!(preprocess_for_tts("1000"), "one thousand"); + assert_eq!( + preprocess_for_tts("1234"), + "one thousand two hundred thirty four" + ); + assert_eq!(preprocess_for_tts("10000"), "ten thousand"); + assert_eq!(preprocess_for_tts("100000"), "one hundred thousand"); + assert_eq!( + preprocess_for_tts("999999"), + "nine hundred ninety nine thousand nine hundred ninety nine" + ); + } + + #[test] + fn expands_times() { + assert_eq!(preprocess_for_tts("11:30"), "eleven thirty"); + assert_eq!(preprocess_for_tts("9:00"), "nine"); + assert_eq!(preprocess_for_tts("9:05"), "nine oh five"); + assert_eq!(preprocess_for_tts("10:09"), "ten oh nine"); + } + + #[test] + fn collapses_whitespace() { + let out = preprocess_for_tts(" hello world "); + assert_eq!(out, "hello world"); + } + + #[test] + fn split_sentences_basic() { + let result = split_sentences("Hello world. How are you? I'm fine!"); + assert_eq!(result, vec!["Hello world.", "How are you?", "I'm fine!"]); + } + + #[test] + fn split_sentences_newline_break() { + let result = split_sentences("First line.\nSecond line."); + assert_eq!(result, vec!["First line.", "Second line."]); + } + + #[test] + fn split_sentences_em_dash_break() { + let result = split_sentences("Start here—then continue."); + assert_eq!(result, vec!["Start here", "then continue."]); + } + + #[test] + fn split_sentences_abbreviations() { + let result = split_sentences("Dr. Smith went home. He was tired."); + assert_eq!(result, vec!["Dr. Smith went home.", "He was tired."]); + } + + #[test] + fn split_sentences_numbered_list() { + let result = split_sentences("1. First item. 2. Second item."); + // "1." and "2." should NOT cause a split (digit before period). + assert_eq!(result, vec!["1. First item.", "2. Second item."]); + } + + #[test] + fn split_sentences_single() { + let result = split_sentences("Just one sentence"); + assert_eq!(result, vec!["Just one sentence"]); + } + + #[test] + fn split_sentences_empty() { + let result = split_sentences(""); + assert_eq!(result, vec![""]); + } + + #[test] + fn filters_trivial_responses() { + assert_eq!(preprocess_for_tts("."), ""); + assert_eq!(preprocess_for_tts(","), ""); + assert_eq!(preprocess_for_tts("!"), ""); + assert_eq!(preprocess_for_tts(" "), ""); + assert_eq!(preprocess_for_tts("ok"), "ok"); + } + + #[test] + fn full_pipeline() { + let input = + "**Agent says:** check https://relay.example.com at 11:30.\n```\nsome code\n```"; + let out = preprocess_for_tts(input); + assert!(!out.contains("**"), "got: {out}"); + assert!(!out.contains("https://"), "got: {out}"); + assert!(out.contains("eleven thirty"), "got: {out}"); + assert!(out.contains("code block omitted"), "got: {out}"); + } + + #[test] + fn preparation_keeps_first_sentence_latency_chunk_separate() { + assert_eq!( + prepare_tts_chunks("First. Second. Third."), + vec!["First.", "Second. Third."] + ); + } + + #[test] + fn shaping_matches_desktop_padding_clamp_and_fade() { + let shaped = shape_tts_chunk(vec![2.0; 400]); + assert_eq!(shaped.len(), 480 + 400 + 1920); + assert!(shaped[..480].iter().all(|sample| *sample == 0.0)); + assert_eq!(shaped[480], 1.0); + assert_eq!(shaped[879], 0.0); + assert!(shaped[880..].iter().all(|sample| *sample == 0.0)); + } + + #[test] + fn shaping_preserves_the_leading_audio_samples() { + let input: Vec = (0..FADE_OUT_SAMPLES * 4) + .map(|index| 0.5 + index as f32 * 1e-4) + .collect(); + let shaped = shape_tts_chunk(input.clone()); + assert_eq!( + &shaped[LEAD_IN_SAMPLES..LEAD_IN_SAMPLES + FADE_OUT_SAMPLES], + &input[..FADE_OUT_SAMPLES] + ); + } + + #[test] + fn shaping_short_and_empty_audio_remains_well_formed() { + let single = shape_tts_chunk(vec![0.5]); + assert_eq!(single[LEAD_IN_SAMPLES], 0.5); + assert_eq!(single.len(), LEAD_IN_SAMPLES + 1 + TRAILING_SILENCE_SAMPLES); + + let empty = shape_tts_chunk(Vec::new()); + assert_eq!(empty.len(), LEAD_IN_SAMPLES + TRAILING_SILENCE_SAMPLES); + assert!(empty.iter().all(|sample| *sample == 0.0)); + } +} diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index c23deef5b2..d0f308e6fc 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1177,6 +1177,7 @@ dependencies = [ "ort", "ort-sys", "rand 0.10.2", + "regex", "sentencepiece-model", "serde", "serde_json", diff --git a/desktop/src-tauri/src/huddle/preprocessing.rs b/desktop/src-tauri/src/huddle/preprocessing.rs index ce85e3145e..ccc3fa0668 100644 --- a/desktop/src-tauri/src/huddle/preprocessing.rs +++ b/desktop/src-tauri/src/huddle/preprocessing.rs @@ -1,667 +1,6 @@ -//! Text preprocessing for TTS output. -//! -//! Mental model: -//! -//! ```text -//! raw agent text -//! → strip fenced code blocks → "code block omitted" -//! → strip inline code → bare text -//! → strip URLs → "link omitted" -//! → strip markdown markers → plain text -//! → strip emoji → (removed) -//! → numbers → words → "forty two" -//! → collapse whitespace → clean string -//! ``` -//! -//! Also provides `split_sentences` — the single sentence-boundary splitter used -//! by both the TTS batching pipeline and the Supertonic text chunker. +//! Compatibility exports for shared TTS preparation. -use regex::Regex; -use std::sync::LazyLock; - -// ── Sentence splitting ──────────────────────────────────────────────────────── - -/// Regex: a sentence-ending punctuation mark followed by whitespace. -static RE_SENTENCE_BOUNDARY: LazyLock = LazyLock::new(|| Regex::new(r"([.!?])\s+").unwrap()); - -/// Common abbreviations that end with a period but are NOT sentence boundaries. -const ABBREVIATIONS: &[&str] = &[ - "Dr.", "Mr.", "Mrs.", "Ms.", "Prof.", "Sr.", "Jr.", "St.", "Ave.", "Rd.", "Blvd.", "Dept.", - "Inc.", "Ltd.", "Co.", "Corp.", "etc.", "vs.", "i.e.", "e.g.", "Ph.D.", -]; - -/// Split text into sentence-sized chunks. -/// -/// Combines regex-based boundary detection with: -/// - Abbreviation awareness (`Dr.`, `Mr.`, etc. don't split) -/// - Digit-before-period check (avoids splitting `1.` `2.` numbered lists) -/// - `\n` and `—` treated as sentence breaks -/// -/// Returns non-empty, trimmed strings. -pub fn split_sentences(text: &str) -> Vec { - // First, split on newlines and em-dashes to get coarse segments. - let coarse: Vec<&str> = text.split(['\n', '—']).collect(); - - let mut sentences = Vec::new(); - - for segment in coarse { - let segment = segment.trim(); - if segment.is_empty() { - continue; - } - // Within each segment, split on sentence-ending punctuation. - let matches: Vec<_> = RE_SENTENCE_BOUNDARY.find_iter(segment).collect(); - if matches.is_empty() { - sentences.push(segment.to_string()); - continue; - } - - let mut last_end = 0usize; - for m in &matches { - let before = &segment[last_end..m.start()]; - let punc_char = &segment[m.start()..m.start() + 1]; - - // Skip if this looks like an abbreviation. - let combined = format!("{}{}", before.trim(), punc_char); - let is_abbrev = ABBREVIATIONS.iter().any(|a| combined.ends_with(a)); - - // Skip if the character before the period is a digit (numbered list). - let is_digit_period = punc_char == "." - && !before.is_empty() - && before.ends_with(|c: char| c.is_ascii_digit()); - - if !is_abbrev && !is_digit_period { - let piece = segment[last_end..m.end()].trim(); - if !piece.is_empty() { - sentences.push(piece.to_string()); - } - last_end = m.end(); - } - } - - if last_end < segment.len() { - let tail = segment[last_end..].trim(); - if !tail.is_empty() { - sentences.push(tail.to_string()); - } - } - } - - if sentences.is_empty() { - vec![text.to_string()] - } else { - sentences - } -} - -// ── Public API ──────────────────────────────────────────────────────────────── - -/// Prepare `text` for TTS synthesis. -/// -/// Applies in order: -/// 1. Fenced code blocks → "code block omitted" -/// 2. Inline code → bare content (backticks stripped) -/// 3. URLs → "link omitted" -/// 4. Markdown bold/italic/underline markers stripped -/// 5. Emoji stripped -/// 6. Numbers → words (integers 0–999, times HH:MM) -/// 7. Excess whitespace collapsed -pub fn preprocess_for_tts(text: &str) -> String { - let s = strip_fenced_code_blocks(text); - let s = strip_inline_code(&s); - let s = strip_urls(&s); - let s = strip_markdown_markers(&s); - let s = strip_emoji(&s); - let s = expand_numbers(&s); - let s = collapse_whitespace(&s); - // Filter trivially short results — ".", ",", etc. would be spoken as - // "period", "comma" by TTS. Agents that have nothing relevant to say - // should not respond at all, but defense-in-depth catches edge cases. - if s.len() <= 1 { - return String::new(); - } - s -} - -// ── Step implementations ────────────────────────────────────────────────────── - -/// Replace fenced code blocks with "code block omitted". -/// -/// Handles both ` ``` ` and `~~~` fences. Multi-line aware. -fn strip_fenced_code_blocks(text: &str) -> String { - let s = replace_fenced(text, "```"); - replace_fenced(&s, "~~~") -} - -fn replace_fenced(text: &str, fence: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut rest = text; - loop { - match rest.find(fence) { - None => { - out.push_str(rest); - break; - } - Some(start) => { - // Everything before the opening fence. - out.push_str(&rest[..start]); - rest = &rest[start + fence.len()..]; - // Skip optional language tag on the same line. - if let Some(nl) = rest.find('\n') { - rest = &rest[nl + 1..]; - } - // Find the closing fence. - match rest.find(fence) { - None => { - // Unclosed fence — treat rest as omitted. - out.push_str(" code block omitted "); - break; - } - Some(end) => { - out.push_str(" code block omitted "); - rest = &rest[end + fence.len()..]; - // Skip trailing newline after closing fence. - if rest.starts_with('\n') { - rest = &rest[1..]; - } - } - } - } - } - } - out -} - -/// Strip backtick-delimited inline code, leaving the inner text. -/// -/// Single-backtick only — triple backtick already handled above. -fn strip_inline_code(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut rest = text; - loop { - match rest.find('`') { - None => { - out.push_str(rest); - break; - } - Some(start) => { - out.push_str(&rest[..start]); - rest = &rest[start + 1..]; - match rest.find('`') { - None => { - // Unclosed — emit as-is. - out.push_str(rest); - break; - } - Some(end) => { - out.push_str(&rest[..end]); - rest = &rest[end + 1..]; - } - } - } - } - } - out -} - -/// Replace http/https URLs with "link omitted". -/// -/// Trailing sentence-ending punctuation (`.`, `!`, `?`) that immediately follows -/// a URL and is at end-of-string or followed by whitespace is preserved so that -/// sentence splitting and TTS prosody are not degraded. -/// -/// Example: `"See https://x.y/z."` → `"See link omitted."` -fn strip_urls(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut rest = text; - loop { - // Find the earliest URL prefix. - let http = rest.find("http://"); - let https = rest.find("https://"); - let url_start = match (http, https) { - (None, None) => { - out.push_str(rest); - break; - } - (Some(a), None) => a, - (None, Some(b)) => b, - (Some(a), Some(b)) => a.min(b), - }; - out.push_str(&rest[..url_start]); - rest = &rest[url_start..]; - // Consume until whitespace or structural delimiter. - let url_end = rest - .find(|c: char| c.is_whitespace() || c == ')' || c == ']' || c == '"' || c == '\'') - .unwrap_or(rest.len()); - let url_token = &rest[..url_end]; - rest = &rest[url_end..]; - - // Check if the URL token ends with sentence-ending punctuation that - // belongs to the surrounding sentence rather than the URL itself. - // A trailing `.`, `!`, or `?` is preserved when it is at end-of-string - // or followed by whitespace (i.e. it is a sentence boundary). - let trailing_punct = if url_token.ends_with(['.', '!', '?']) { - let after = rest; // rest is already past url_end - if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { - // Preserve the trailing punctuation. - &url_token[url_token.len() - 1..] - } else { - "" - } - } else { - "" - }; - - out.push_str("link omitted"); - out.push_str(trailing_punct); - } - out -} - -/// Strip `**`, `*`, `__`, `_emphasis_`, `~~` markdown markers. -/// -/// Underscores are only stripped when they wrap a word (`_text_`). -/// Standalone underscores (e.g. `snake_case` identifiers) are preserved. -fn strip_markdown_markers(text: &str) -> String { - // Order matters: strip multi-char markers before single-char. - let s = text.replace("**", ""); - let s = s.replace("__", ""); - let s = s.replace("~~", ""); - let s = s.replace('*', ""); - strip_underscore_emphasis(&s) -} - -/// Strip `_text_` emphasis markers while preserving underscores in identifiers. -/// -/// A `_` is treated as an emphasis delimiter only when it is preceded by -/// whitespace or the start of the string AND followed by a non-whitespace char, -/// or vice-versa for the closing delimiter. -fn strip_underscore_emphasis(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let chars: Vec = text.chars().collect(); - let len = chars.len(); - let mut i = 0; - while i < len { - if chars[i] == '_' { - // Opening delimiter: preceded by whitespace/start, followed by non-whitespace. - let prev_is_boundary = i == 0 || chars[i - 1].is_whitespace(); - let next_is_nonspace = i + 1 < len && !chars[i + 1].is_whitespace(); - if prev_is_boundary && next_is_nonspace { - // Look for a matching closing `_`. - if let Some(close) = (i + 1..len).find(|&j| { - chars[j] == '_' - && !chars[j - 1].is_whitespace() - && (j + 1 >= len - || chars[j + 1].is_whitespace() - || chars[j + 1].is_ascii_punctuation()) - }) { - // Emit the inner text without the delimiters. - for &ch in &chars[i + 1..close] { - out.push(ch); - } - i = close + 1; - continue; - } - } - // Not an emphasis delimiter — emit as-is. - out.push('_'); - } else { - out.push(chars[i]); - } - i += 1; - } - out -} - -/// Strip Unicode emoji (characters in common emoji ranges). -/// -/// Covers the main Emoji block (U+1F300–U+1FAFF) and supplemental ranges. -/// ASCII emoticons like `:)` are left as-is. -fn strip_emoji(text: &str) -> String { - text.chars().filter(|&c| !is_emoji(c)).collect() -} - -#[inline] -fn is_emoji(c: char) -> bool { - matches!(c, - '\u{1F300}'..='\u{1FAFF}' // Misc symbols, emoticons, transport, etc. - | '\u{2600}'..='\u{27BF}' // Misc symbols, dingbats - | '\u{FE00}'..='\u{FE0F}' // Variation selectors - | '\u{1F000}'..='\u{1F02F}'// Mahjong/domino tiles - | '\u{1F0A0}'..='\u{1F0FF}'// Playing cards - | '\u{200D}' // Zero-width joiner (used in emoji sequences) - | '\u{20E3}' // Combining enclosing keycap - ) -} - -/// Expand numbers to spoken words. -/// -/// Handles: -/// - Times: `HH:MM` → "eleven thirty" -/// - Integers 0–999,999 -/// - Leaves other numeric strings (e.g. "3.14", "1000000+") as-is. -fn expand_numbers(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut chars = text.char_indices().peekable(); - - while let Some((i, c)) = chars.next() { - if c.is_ascii_digit() { - // Collect the full token (digits, colon, dots). - let start = i; - let mut end = i + c.len_utf8(); - while let Some(&(j, nc)) = chars.peek() { - if nc.is_ascii_digit() || nc == ':' || nc == '.' { - end = j + nc.len_utf8(); - chars.next(); - } else { - break; - } - } - let token = &text[start..end]; - out.push_str(&expand_numeric_token(token)); - } else { - out.push(c); - } - } - out -} - -fn expand_numeric_token(token: &str) -> String { - // Strip trailing punctuation that the token collector may have included - // (e.g. "11:30." from "at 11:30.") before attempting to parse. - let token = token.trim_end_matches(|c: char| !c.is_ascii_digit()); - - // Time: HH:MM - if let Some(colon) = token.find(':') { - let h = &token[..colon]; - let m = &token[colon + 1..]; - if let (Ok(hh), Ok(mm)) = (h.parse::(), m.parse::()) { - if hh < 24 && mm < 60 { - let hour_word = int_to_words(hh); - let min_word = if mm == 0 { - String::new() - } else if mm < 10 { - // "9:05" → "nine oh five" (not "nine five") - format!(" oh {}", int_to_words(mm)) - } else { - format!(" {}", int_to_words(mm)) - }; - return format!("{}{}", hour_word, min_word); - } - } - // Not a valid time — return as-is. - return token.to_string(); - } - - // Plain integer 0–999,999. - if token.chars().all(|c| c.is_ascii_digit()) { - if let Ok(n) = token.parse::() { - if n <= 999_999 { - return int_to_words(n); - } - } - } - - // Anything else (decimals, millions+) — leave as-is. - token.to_string() -} - -/// Convert an integer 0–999,999 to English words. -fn int_to_words(n: u32) -> String { - const ONES: &[&str] = &[ - "zero", - "one", - "two", - "three", - "four", - "five", - "six", - "seven", - "eight", - "nine", - "ten", - "eleven", - "twelve", - "thirteen", - "fourteen", - "fifteen", - "sixteen", - "seventeen", - "eighteen", - "nineteen", - ]; - const TENS: &[&str] = &[ - "", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety", - ]; - - if n < 20 { - return ONES[n as usize].to_string(); - } - if n < 100 { - let ten = TENS[(n / 10) as usize]; - let one = n % 10; - return if one == 0 { - ten.to_string() - } else { - format!("{} {}", ten, ONES[one as usize]) - }; - } - if n < 1000 { - let hundreds = n / 100; - let remainder = n % 100; - let hundred_word = format!("{} hundred", ONES[hundreds as usize]); - return if remainder == 0 { - hundred_word - } else { - format!("{} {}", hundred_word, int_to_words(remainder)) - }; - } - // 1,000–999,999 - let thousands = n / 1000; - let remainder = n % 1000; - let thousand_word = format!("{} thousand", int_to_words(thousands)); - if remainder == 0 { - thousand_word - } else { - format!("{} {}", thousand_word, int_to_words(remainder)) - } -} - -/// Collapse runs of whitespace (spaces, tabs, newlines) to a single space. -/// Trims leading/trailing whitespace. -fn collapse_whitespace(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut prev_space = true; // Start true to trim leading whitespace. - for c in text.chars() { - if c.is_whitespace() { - if !prev_space { - out.push(' '); - prev_space = true; - } - } else { - out.push(c); - prev_space = false; - } - } - // Trim trailing space. - if out.ends_with(' ') { - out.pop(); - } - out -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn strips_fenced_code_block() { - let input = "Here is some code:\n```rust\nfn main() {}\n```\nDone."; - let out = preprocess_for_tts(input); - assert!(out.contains("code block omitted"), "got: {out}"); - assert!(!out.contains("fn main"), "got: {out}"); - } - - #[test] - fn strips_inline_code() { - let out = preprocess_for_tts("Call `foo()` now."); - assert_eq!(out, "Call foo() now."); - } - - #[test] - fn strips_urls() { - let out = preprocess_for_tts("See https://example.com for details."); - assert!(out.contains("link omitted"), "got: {out}"); - assert!(!out.contains("example.com"), "got: {out}"); - } - - #[test] - fn strips_url_preserves_trailing_period() { - // Trailing `.` at end of sentence must be preserved for sentence splitting. - let out = strip_urls("See https://x.y/z."); - assert_eq!(out, "See link omitted.", "got: {out}"); - } - - #[test] - fn strips_url_preserves_trailing_exclamation() { - let out = strip_urls("Visit https://example.com!"); - assert_eq!(out, "Visit link omitted!", "got: {out}"); - } - - #[test] - fn strips_url_preserves_trailing_question() { - let out = strip_urls("Did you see https://example.com?"); - assert_eq!(out, "Did you see link omitted?", "got: {out}"); - } - - #[test] - fn strips_url_mid_sentence_no_punct_preserved() { - // URL in the middle of a sentence — no trailing punct to preserve. - let out = strip_urls("Check https://example.com for more info."); - assert_eq!(out, "Check link omitted for more info.", "got: {out}"); - } - - #[test] - fn strips_bold_italic() { - let out = preprocess_for_tts("**bold** and *italic* and _under_"); - assert_eq!(out, "bold and italic and under"); - } - - #[test] - fn preserves_standalone_underscores() { - // snake_case identifiers should not be mangled. - let out = preprocess_for_tts("call foo_bar() or baz_qux"); - assert!(out.contains("foo_bar"), "got: {out}"); - assert!(out.contains("baz_qux"), "got: {out}"); - } - - #[test] - fn strips_tilde_fenced_block() { - let input = "Here:\n~~~python\nprint('hi')\n~~~\nDone."; - let out = preprocess_for_tts(input); - assert!(out.contains("code block omitted"), "got: {out}"); - assert!(!out.contains("print"), "got: {out}"); - } - - #[test] - fn expands_integers() { - assert_eq!(preprocess_for_tts("42"), "forty two"); - assert_eq!(preprocess_for_tts("0"), "zero"); - assert_eq!(preprocess_for_tts("11"), "eleven"); - assert_eq!(preprocess_for_tts("100"), "one hundred"); - } - - #[test] - fn expands_thousands() { - assert_eq!(preprocess_for_tts("1000"), "one thousand"); - assert_eq!( - preprocess_for_tts("1234"), - "one thousand two hundred thirty four" - ); - assert_eq!(preprocess_for_tts("10000"), "ten thousand"); - assert_eq!(preprocess_for_tts("100000"), "one hundred thousand"); - assert_eq!( - preprocess_for_tts("999999"), - "nine hundred ninety nine thousand nine hundred ninety nine" - ); - } - - #[test] - fn expands_times() { - assert_eq!(preprocess_for_tts("11:30"), "eleven thirty"); - assert_eq!(preprocess_for_tts("9:00"), "nine"); - assert_eq!(preprocess_for_tts("9:05"), "nine oh five"); - assert_eq!(preprocess_for_tts("10:09"), "ten oh nine"); - } - - #[test] - fn collapses_whitespace() { - let out = preprocess_for_tts(" hello world "); - assert_eq!(out, "hello world"); - } - - #[test] - fn split_sentences_basic() { - let result = split_sentences("Hello world. How are you? I'm fine!"); - assert_eq!(result, vec!["Hello world.", "How are you?", "I'm fine!"]); - } - - #[test] - fn split_sentences_newline_break() { - let result = split_sentences("First line.\nSecond line."); - assert_eq!(result, vec!["First line.", "Second line."]); - } - - #[test] - fn split_sentences_em_dash_break() { - let result = split_sentences("Start here—then continue."); - assert_eq!(result, vec!["Start here", "then continue."]); - } - - #[test] - fn split_sentences_abbreviations() { - let result = split_sentences("Dr. Smith went home. He was tired."); - assert_eq!(result, vec!["Dr. Smith went home.", "He was tired."]); - } - - #[test] - fn split_sentences_numbered_list() { - let result = split_sentences("1. First item. 2. Second item."); - // "1." and "2." should NOT cause a split (digit before period). - assert_eq!(result, vec!["1. First item.", "2. Second item."]); - } - - #[test] - fn split_sentences_single() { - let result = split_sentences("Just one sentence"); - assert_eq!(result, vec!["Just one sentence"]); - } - - #[test] - fn split_sentences_empty() { - let result = split_sentences(""); - assert_eq!(result, vec![""]); - } - - #[test] - fn filters_trivial_responses() { - assert_eq!(preprocess_for_tts("."), ""); - assert_eq!(preprocess_for_tts(","), ""); - assert_eq!(preprocess_for_tts("!"), ""); - assert_eq!(preprocess_for_tts(" "), ""); - assert_eq!(preprocess_for_tts("ok"), "ok"); - } - - #[test] - fn full_pipeline() { - let input = - "**Agent says:** check https://relay.example.com at 11:30.\n```\nsome code\n```"; - let out = preprocess_for_tts(input); - assert!(!out.contains("**"), "got: {out}"); - assert!(!out.contains("https://"), "got: {out}"); - assert!(out.contains("eleven thirty"), "got: {out}"); - assert!(out.contains("code block omitted"), "got: {out}"); - } -} +#[allow(unused_imports)] +pub use buzz_voice_pkg::preparation::{ + prepare_tts_chunks, preprocess_for_tts, shape_tts_chunk, split_sentences, +}; diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 63a435cd8e..3592093ae5 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -47,7 +47,9 @@ use std::{ }; use super::pocket::{load_text_to_speech, load_voice_style, SAMPLE_RATE, VOICE_FILE_EXT}; -use super::preprocessing::{preprocess_for_tts, split_sentences}; +use super::preprocessing::{prepare_tts_chunks, shape_tts_chunk}; +#[cfg(test)] +use buzz_voice_pkg::preparation::group_sentences_into_chunks; // ── Constants ───────────────────────────────────────────────────────────────── @@ -69,51 +71,6 @@ const MONITOR_TICK: Duration = Duration::from_millis(10); /// Pocket TTS is a one-step consistency model, not diffusion. Kept for API compat. const SYNTH_STEPS: usize = 1; -/// Fade-out length in samples (8 ms at 24 kHz ≈ 192 samples). -/// -/// Applied only at the *end* of each synthesised sentence to eliminate the -/// click that would otherwise occur when a non-zero waveform terminates -/// abruptly. **No fade-in is applied** — see `apply_fade_out` for the -/// rationale and `examples/pocket_onset_probe.rs` for the measurement that -/// motivated removing the leading fade. -const FADE_OUT_SAMPLES: usize = (SAMPLE_RATE as f64 * 0.008) as usize; - -/// Length of the zero-sample cushion prepended before each synthesized -/// sentence chunk, so the OS audio device / rodio mixer has a fully-quiet -/// ramp-up window before the real onset hits. -/// -/// This used to be applied only before the first sentence of a whole response. -/// That still left later sentence chunks vulnerable to first-syllable clipping -/// when their first phoneme was soft (notably `I'm` / `I've`) and rodio crossed -/// from an explicit silence buffer straight into non-zero speech. 20 ms ≈ 480 -/// samples is enough to cover a CoreAudio buffer turnover without being audible -/// as latency. At sentence boundaries this lead-in is budgeted out of the -/// existing inter-sentence pause, so it does not lengthen multi-sentence gaps. -const SENTENCE_LEAD_IN_SAMPLES: usize = (SAMPLE_RATE as f64 * 0.020) as usize; - -/// Approximate character budget for one synthesis chunk. -/// -/// Upstream pocket-tts groups sentences into chunks of up to -/// `MAX_TOKEN_PER_CHUNK = 50` tokenizer tokens (`default_parameters.py`) — -/// typically multi-sentence chunks — because every `generate()` call is an -/// independent generation with a cold FlowLM start, and each chunk boundary -/// is an exposed prosody seam (kyutai-labs/pocket-tts #151; the Kyutai team -/// names chunk stitching as the reliability lever). Our previous -/// sentence-per-call path created ~2–4× more seams than upstream. -/// -/// We don't ship the SentencePiece tokenizer, so 50 tokens is approximated -/// with a character budget. The bundled 4k-entry vocab averages ~4 chars per -/// token, but usage-weighted English text leans on short common tokens, so -/// the effective ratio is ~2–4 chars/token and 200 chars ≈ 60–100 tokens — -/// modestly above upstream's 50, deliberately: erring large means fewer -/// seams, and even ~100 tokens is far below the model's 500-LM-step (~40 s) -/// ceiling. Do not shrink this budget to chase an exact 50-token match. -const MAX_CHUNK_CHARS: usize = 200; - -/// Silence inserted between sentences by the TTS pipeline (seconds). -/// Injected as a silent buffer between each synthesized sentence chunk. -const INTER_SENTENCE_SILENCE: f32 = 0.1; - // ── Public pipeline handle ──────────────────────────────────────────────────── /// Handle to the running TTS pipeline. @@ -417,11 +374,9 @@ fn tts_worker( // `tts_active` lifecycle: set on the first append while idle, cleared // whenever the player has fully drained — either in the idle timeout // arm or on item receipt before synthesis begins. - let silence_buf_len = (INTER_SENTENCE_SILENCE * SAMPLE_RATE as f32) as usize; // `first_append` = "no audio queued since the player last went idle". - // Flipped by `build_sentence_append_buffer` on the first real append; the - // idle branch below uses it to decide when to drop `tts_active` and to - // arm a fresh lead-in cushion for the next utterance. + // The first successful append clears it; the idle branch below uses it to + // decide when to drop `tts_active`. let mut first_append = true; loop { @@ -485,24 +440,14 @@ fn tts_worker( first_append = true; } - // Preprocess text. - let text = preprocess_for_tts(&raw_text); - if text.is_empty() { - continue; - } - - // Split into sentences, then group into synthesis chunks: the first + // Prepare and group synthesis chunks: the first // sentence stays alone (fast time-to-first-audio), the rest pack // greedily up to MAX_CHUNK_CHARS. Each chunk is one `generate()` // call; playback of chunk N overlaps synthesis of chunk N+1 // (lookahead pipelining). Grouping matches upstream's ~50-token // chunking and halves the exposed prosody seams on multi-sentence // replies — see MAX_CHUNK_CHARS. - let sentences: Vec = split_sentences(&text) - .into_iter() - .filter(|s| !s.trim().is_empty()) - .collect(); - let chunks = group_sentences_into_chunks(&sentences, MAX_CHUNK_CHARS); + let chunks = prepare_tts_chunks(&raw_text); for chunk in &chunks { if handle_cancel_or_shutdown( @@ -523,19 +468,13 @@ fn tts_worker( match engine.synth_chunk(text, "en", &style, SYNTH_STEPS) { Ok(samples) if !samples.is_empty() => { - let mut audio = clamp_to_full_scale(samples); - // Fade-out only — fading-in would attenuate the consonant - // onset (see `apply_fade_out` docstring + the - // 2026-05-18 "first little sound is missing" regression). - apply_fade_out(&mut audio); - // Build one contiguous buffer per synthesized sentence: // lead-in cushion + audio + trailing gap. Keeping this as // a single rodio source preserves the original queue/drain // semantics (one append per sentence) while still giving // every chunk a quiet device warm-up window. - let buf = - build_sentence_append_buffer(&mut first_append, audio, silence_buf_len); + let buf = shape_tts_chunk(samples); + first_append = false; // Check-and-append under `player_ops`, serialized with // the monitor: a barge-in may have arrived during @@ -644,130 +583,6 @@ fn lock_player_ops(ops: &Mutex<()>) -> MutexGuard<'_, ()> { ops.lock().unwrap_or_else(PoisonError::into_inner) } -/// Hard-clamp samples to ±1.0 full scale. -/// -/// No gain is applied: Pocket TTS already emits speech-level audio -/// (peaks 0.4–0.97, RMS ≈ −20 dBFS across varied sentences — measured by -/// `examples/pocket_clip_probe`), matching the kyutai reference pipeline, -/// which applies no output scaling. Two earlier gain stages were both -/// regressions against that baseline: per-sentence peak normalization caused -/// level pumping between sentences, and the fixed 9.3× gain that replaced it -/// was calibrated on a single anomalously-quiet bench utterance (peak 0.076) -/// and clipped 13–34% of samples on real speech ("blown out", 2026-06-12). -/// The clamp alone remains as the safety net against outlier transients. -fn clamp_to_full_scale(samples: Vec) -> Vec { - samples.into_iter().map(|s| s.clamp(-1.0, 1.0)).collect() -} - -/// Apply a short linear fade-out at the *end* of `samples`. -/// -/// Uses `FADE_OUT_SAMPLES` (8 ms) or half the buffer length, whichever is -/// smaller. Eliminates the click that occurs when a non-zero waveform -/// terminates abruptly at a sentence boundary. -/// -/// # Why no fade-in -/// -/// An earlier revision (pre 2026-05) symmetrically faded *in* over the same -/// 8 ms window. That swallowed the leading consonant attack on every -/// sentence — Pocket TTS produces real audio energy inside the first -/// millisecond (RMS ≈ 0.02, peak ≈ 0.03 measured across four prompts in -/// `examples/pocket_onset_probe.rs`), and a linear 0→1 ramp over 192 samples -/// scales those onset samples by ≤50 % for the first ~4 ms. The result was -/// the "first little sound or two is missing" regression heard on -/// 2026-05-18. -/// -/// The first sample of Pocket output measures ≈ 0.0018 (≈ −54 dBFS) — well -/// below the threshold at which a DC-jump would be audible as a click — so -/// no fade-in is needed. The OS audio device gets its quiet ramp-up window -/// from `SENTENCE_LEAD_IN_SAMPLES` instead, inserted as pure silence before -/// each sentence buffer. -fn apply_fade_out(samples: &mut [f32]) { - let len = samples.len(); - let fade = FADE_OUT_SAMPLES.min(len / 2); - for i in 0..fade { - samples[len - 1 - i] *= i as f32 / fade as f32; - } -} - -/// Build the single buffer appended to the rodio `Player` for one synthesised -/// sentence. -/// -/// Every sentence chunk gets a short lead-in pad immediately before its audio. -/// This matters for chunks that start with soft first phonemes (`I'm`, `I've`): -/// the synthesized buffer can begin with speech within the first millisecond, -/// so the playback layer must provide the device/mixer cushion. -/// To keep the audible gap unchanged, the trailing silence after this chunk is -/// shortened by the same amount (`silence_buf_len - SENTENCE_LEAD_IN_SAMPLES`): -/// sentence N contributes 80 ms of post-speech silence and sentence N+1 -/// contributes the remaining 20 ms of pre-speech cushion. -/// -/// The lead-in, audio, and trailing silence are concatenated into one -/// `SamplesBuffer` before appending. This keeps rodio's queue shape at one -/// tracked source per synthesized sentence, avoiding source-boundary/drain -/// regressions from enqueueing the lead-in, audio, and tail as separate sounds. -/// -/// `first_append` is flipped on the first call after the player goes idle. -/// The worker uses it in the idle branch of the main loop to distinguish -/// "never queued anything since last drain" from "drained after speaking", -/// which controls when `tts_active` is released and the lead-in re-armed. -fn build_sentence_append_buffer( - first_append: &mut bool, - audio: Vec, - silence_buf_len: usize, -) -> Vec { - if *first_append { - *first_append = false; - } - - let trailing_silence_len = silence_buf_len.saturating_sub(SENTENCE_LEAD_IN_SAMPLES); - let mut buf = Vec::with_capacity(SENTENCE_LEAD_IN_SAMPLES + audio.len() + trailing_silence_len); - buf.extend(std::iter::repeat_n(0.0_f32, SENTENCE_LEAD_IN_SAMPLES)); - buf.extend(audio); - buf.extend(std::iter::repeat_n(0.0_f32, trailing_silence_len)); - buf -} - -/// Group sentences into synthesis chunks. -/// -/// The first sentence always stands alone — it is what the listener hears -/// first, and synthesizing it by itself keeps time-to-first-audio at the -/// single-sentence cost. Subsequent sentences pack greedily: a sentence -/// joins the current chunk while the combined length stays within -/// `max_chars`; otherwise it starts a new chunk. A single sentence longer -/// than `max_chars` becomes its own chunk unsplit — Pocket TTS handles long -/// single sentences fine (the ceiling is the 500-LM-step default), it's the -/// *seams* we're minimizing. -/// -/// Sentences within a chunk are joined with a single space; sentence-ending -/// punctuation is preserved by `split_sentences`, so the model sees natural -/// multi-sentence prose — the same shape upstream's ~50-token chunker feeds it. -fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec { - let mut chunks: Vec = Vec::new(); - for (i, sentence) in sentences.iter().enumerate() { - let sentence = sentence.trim(); - if sentence.is_empty() { - continue; - } - if i == 0 || chunks.is_empty() { - chunks.push(sentence.to_string()); - continue; - } - // Never merge into the first chunk — it's the latency-critical one. - let can_merge = chunks.len() > 1 - && chunks - .last() - .is_some_and(|c| c.len() + 1 + sentence.len() <= max_chars); - if can_merge { - let last = chunks.last_mut().expect("non-empty checked above"); - last.push(' '); - last.push_str(sentence); - } else { - chunks.push(sentence.to_string()); - } - } - chunks -} - // drain_until_shutdown lives in super (huddle/mod.rs) — shared with stt.rs. use super::drain_until_shutdown; diff --git a/desktop/src-tauri/src/huddle/tts_tests.rs b/desktop/src-tauri/src/huddle/tts_tests.rs index 7887f8bbdb..90d9b147a7 100644 --- a/desktop/src-tauri/src/huddle/tts_tests.rs +++ b/desktop/src-tauri/src/huddle/tts_tests.rs @@ -736,186 +736,6 @@ fn frames_while_tts_inactive_are_not_counted() { ); } -// ── apply_fade_out tests ────────────────────────────────────────────────── - -/// The fade-out half of the old `apply_fades`: last sample is silenced -/// and the ramp is monotonic. Mid-buffer must be untouched. -#[test] -fn apply_fade_out_short_buffer() { - let mut samples = vec![1.0f32; 10]; - apply_fade_out(&mut samples); - assert_eq!(samples[9], 0.0, "last sample should be silenced"); - assert!(samples[5] > 0.5, "mid-buffer should be near-untouched"); -} - -/// REGRESSION (2026-05-18): the *first* samples must NOT be attenuated. -/// An earlier `apply_fades` symmetrically faded in over 8 ms which -/// swallowed the consonant onset of every sentence. -/// Lock in: samples[0..FADE_OUT_SAMPLES] are byte-equal to input. -#[test] -fn apply_fade_out_does_not_touch_leading_samples() { - // Input long enough that fade window doesn't overlap (≫ 2× fade). - let n = FADE_OUT_SAMPLES * 4; - let input: Vec = (0..n).map(|i| 0.5 + (i as f32) * 1e-4).collect(); - let mut samples = input.clone(); - apply_fade_out(&mut samples); - for i in 0..FADE_OUT_SAMPLES { - assert_eq!( - samples[i], input[i], - "leading sample {i} must not be attenuated (was {} → {})", - input[i], samples[i] - ); - } - // And the trailing fade still works. - assert_eq!(samples[n - 1], 0.0); -} - -#[test] -fn apply_fade_out_empty_buffer() { - let mut samples: Vec = vec![]; - apply_fade_out(&mut samples); - assert!(samples.is_empty()); -} - -#[test] -fn apply_fade_out_single_sample() { - // fade = min(FADE_OUT_SAMPLES, len/2) = 0, so nothing changes. - let mut samples = vec![1.0f32]; - apply_fade_out(&mut samples); - assert_eq!(samples[0], 1.0); -} - -/// Sanity-check the per-sentence cushion length: 20 ms at 24 kHz must -/// land at exactly 480 samples. This is a const computation, so the -/// real value of this test is documenting *why* 20 ms was chosen — it -/// covers a typical CoreAudio buffer turnover (256–1024 samples) -/// without being audible as user-facing latency. -#[test] -fn sentence_lead_in_is_sane() { - assert_eq!(SENTENCE_LEAD_IN_SAMPLES, 480, "20 ms × 24 kHz"); -} - -// ── build_sentence_append_buffer tests ─────────────────────────────────── - -/// REGRESSION: every chunk needs an onset cushion; synthesized chunks -/// can start with speech energy within the first millisecond. -#[test] -fn lead_in_pad_is_present_for_every_sentence_chunk() { - const SENTENCE_AUDIO_LEN: usize = 1000; - const SILENCE_BUF_LEN: usize = 2400; // 100 ms at 24 kHz, like production - const N_SENTENCES: usize = 5; - - let mut first = true; - - for _ in 0..N_SENTENCES { - let buf = build_sentence_append_buffer( - &mut first, - vec![0.5_f32; SENTENCE_AUDIO_LEN], - SILENCE_BUF_LEN, - ); - - assert_eq!(buf.len(), SENTENCE_AUDIO_LEN + SILENCE_BUF_LEN); - assert!( - buf[..SENTENCE_LEAD_IN_SAMPLES].iter().all(|&s| s == 0.0), - "lead-in pad must be pure silence" - ); - assert!( - buf[SENTENCE_LEAD_IN_SAMPLES..SENTENCE_LEAD_IN_SAMPLES + SENTENCE_AUDIO_LEN] - .iter() - .all(|&s| s == 0.5), - "sentence audio must immediately follow the lead-in" - ); - assert!( - buf[SENTENCE_LEAD_IN_SAMPLES + SENTENCE_AUDIO_LEN..] - .iter() - .all(|&s| s == 0.0), - "trailing gap must be pure silence" - ); - } - - assert!(!first, "first_append flag must be cleared after first call"); -} - -/// `first_append` still flips on the first call for `tts_active` gating. -#[test] -fn build_sentence_append_buffer_flips_first_append() { - let mut first = true; - let _ = build_sentence_append_buffer(&mut first, vec![0.5; 100], 2400); - assert!(!first, "first call must flip the flag"); - - // Subsequent call: still has a per-sentence lead-in, flag stays false. - let buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], 2400); - assert!(buf[..SENTENCE_LEAD_IN_SAMPLES].iter().all(|&s| s == 0.0)); - assert!(!first); -} - -/// Leading silence is exactly the lead-in; no pre-audio gap is double-counted. -#[test] -fn first_sentence_leading_silence_is_exactly_lead_in() { - let mut first = true; - let buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], 2400); - assert!(buf[..SENTENCE_LEAD_IN_SAMPLES].iter().all(|&s| s == 0.0)); - assert_eq!(buf[SENTENCE_LEAD_IN_SAMPLES], 0.5); -} - -/// Tail silence plus the next lead-in preserves the 100 ms sentence gap. -#[test] -fn sentence_gap_budget_is_preserved() { - let mut first = true; - let silence_buf_len = 2400; - let first_buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], silence_buf_len); - let second_buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], silence_buf_len); - - let first_tail = &first_buf[SENTENCE_LEAD_IN_SAMPLES + 100..]; - let second_lead = &second_buf[..SENTENCE_LEAD_IN_SAMPLES]; - assert_eq!(first_tail.len(), silence_buf_len - SENTENCE_LEAD_IN_SAMPLES); - assert_eq!(second_lead.len(), SENTENCE_LEAD_IN_SAMPLES); - assert_eq!(first_tail.len() + second_lead.len(), silence_buf_len); -} - -/// Regression guard: one contiguous rodio source per synthesized sentence. -#[test] -fn sentence_append_buffer_is_one_contiguous_source() { - let mut first = true; - let buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], 2400); - - assert_eq!(buf.len(), 2400 + 100); - assert!(buf[..SENTENCE_LEAD_IN_SAMPLES].iter().all(|&s| s == 0.0)); - assert!( - buf[SENTENCE_LEAD_IN_SAMPLES..SENTENCE_LEAD_IN_SAMPLES + 100] - .iter() - .all(|&s| s == 0.5) - ); -} - -// ── clamp_to_full_scale tests ───────────────────────────────────────────── - -/// In-range speech audio passes through bit-exact — no gain is applied. -/// (Pocket output is already at speech level; see the fn doc-comment for -/// the history of the two gain-stage regressions this replaced.) -#[test] -fn clamp_to_full_scale_passes_speech_audio_unchanged() { - let input = vec![0.42_f32, -0.97, 0.076, 0.0]; - let out = clamp_to_full_scale(input.clone()); - assert_eq!(out, input); -} - -/// Outlier transients beyond full scale are hard-clamped to ±1.0 rather -/// than wrapping. -#[test] -fn clamp_to_full_scale_clamps_outliers() { - let input = vec![1.5_f32, -2.0, 0.5]; - let out = clamp_to_full_scale(input); - assert_eq!(out, vec![1.0, -1.0, 0.5]); -} - -/// Empty input round-trips to empty output. -#[test] -fn clamp_to_full_scale_empty_buffer() { - let out = clamp_to_full_scale(Vec::new()); - assert!(out.is_empty()); -} - // ── group_sentences_into_chunks tests ───────────────────────────────────── fn s(v: &[&str]) -> Vec { diff --git a/mobile/android/app/build.gradle.kts b/mobile/android/app/build.gradle.kts index 501a31c46f..3b0b966ece 100644 --- a/mobile/android/app/build.gradle.kts +++ b/mobile/android/app/build.gradle.kts @@ -92,8 +92,15 @@ android { versionName = flutter.versionName testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" resValue("string", "app_name", "Buzz") + ndk { + // Native Pocket voice artifacts are built for device arm64 and + // emulator x86_64. Do not publish an installable ABI without them. + abiFilters += listOf("arm64-v8a", "x86_64") + } } + sourceSets["main"].jniLibs.srcDir("../../.generated/voice/android/jniLibs") + signingConfigs { if (hasUploadSigning) { create("upload") { @@ -124,6 +131,20 @@ android { } } +val buildBuzzVoiceNative by tasks.registering(Exec::class) { + workingDir(rootProject.projectDir.parentFile.parentFile) + environment("ANDROID_NDK_HOME", android.ndkDirectory.absolutePath) + commandLine( + "/bin/bash", + "-c", + ". ./bin/activate-hermit && ./scripts/mobile-voice-native.sh build-android", + ) +} + +tasks.named("preBuild").configure { + dependsOn(buildBuzzVoiceNative) +} + dependencies { testImplementation(kotlin("test")) diff --git a/mobile/android/app/src/main/AndroidManifest.xml b/mobile/android/app/src/main/AndroidManifest.xml index 17a16742af..b63e4d40ba 100644 --- a/mobile/android/app/src/main/AndroidManifest.xml +++ b/mobile/android/app/src/main/AndroidManifest.xml @@ -4,6 +4,8 @@ + if (shouldStopPocketVoiceForAudioFocusChange(change)) { + stopAndNotify("interrupted") + } + } + private val noisyReceiver = + object : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + stopAndNotify("routeLost") + } + } + + init { + channel.setMethodCallHandler(::handle) + applicationContext.registerReceiver( + noisyReceiver, + IntentFilter(AudioManager.ACTION_AUDIO_BECOMING_NOISY), + ) + } + + fun background() { + mainHandler.post { + stop(notify = null) + channel.invokeMethod("backgrounded", null) + } + } + + fun dispose() { + stop(notify = null) + channel.setMethodCallHandler(null) + runCatching { applicationContext.unregisterReceiver(noisyReceiver) } + } + + private fun handle( + call: MethodCall, + result: MethodChannel.Result, + ) { + when (call.method) { + "play" -> { + val pcm = call.argument("pcm") + val sampleRate = call.argument("sampleRate") + if (pcm == null || sampleRate == null) { + result.error("invalid_arguments", "Expected PCM and sample rate.", null) + return + } + runCatching { play(pcm, sampleRate) } + .onSuccess { result.success(null) } + .onFailure { + result.error("playback_failed", it.message, null) + } + } + "stop" -> { + stop(notify = null) + result.success(null) + } + "availableCapacity" -> { + val path = call.arguments as? String + if (path == null) { + result.error("invalid_arguments", "Expected a storage path.", null) + } else { + runCatching { StatFs(path).availableBytes } + .onSuccess(result::success) + .onFailure { result.error("storage_failed", it.message, null) } + } + } + "excludeFromBackup" -> result.success(null) + else -> result.notImplemented() + } + } + + private fun play( + pcm: ByteArray, + sampleRate: Int, + ) { + stop(notify = null) + check(requestFocus()) { "Audio focus was denied." } + val nextTrack = + try { + val attributes = + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_ASSISTANCE_ACCESSIBILITY) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + val format = + AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(sampleRate) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .build() + val minimum = AudioTrack.getMinBufferSize( + sampleRate, + AudioFormat.CHANNEL_OUT_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ) + AudioTrack.Builder() + .setAudioAttributes(attributes) + .setAudioFormat(format) + .setTransferMode(AudioTrack.MODE_STREAM) + .setBufferSizeInBytes(max(minimum, 32 * 1024)) + .build() + .also { + if (it.state != AudioTrack.STATE_INITIALIZED) { + it.release() + error("Unable to initialize voice audio.") + } + } + } catch (error: Exception) { + abandonFocus() + throw error + } + track = nextTrack + val currentGeneration = generation.incrementAndGet() + try { + nextTrack.play() + } catch (error: Exception) { + track = null + nextTrack.release() + abandonFocus() + throw error + } + thread(name = "buzz-pocket-playback") { + var offset = 0 + var complete = false + var started = false + fun reportStartedIfPlaybackAdvanced() { + if (started || generation.get() != currentGeneration) return + val playedFrames = + runCatching { + nextTrack.playbackHeadPosition.toLong() and 0xFFFF_FFFFL + }.getOrNull() ?: return + if (playedFrames == 0L) return + started = true + mainHandler.post { + if (generation.get() == currentGeneration && track === nextTrack) { + channel.invokeMethod("started", null) + } + } + } + try { + while (offset < pcm.size && generation.get() == currentGeneration) { + val written = nextTrack.write( + pcm, + offset, + pcm.size - offset, + AudioTrack.WRITE_BLOCKING, + ) + if (written <= 0) break + offset += written + reportStartedIfPlaybackAdvanced() + } + complete = offset == pcm.size + if (complete) { + val targetFrames = pcm.size.toLong() / 2 + val deadlineNanos = + System.nanoTime() + + (targetFrames * 1_000_000_000L / sampleRate) + + 2_000_000_000L + var drained = false + while ( + generation.get() == currentGeneration && + System.nanoTime() < deadlineNanos + ) { + val playedFrames = + runCatching { + nextTrack.playbackHeadPosition.toLong() and 0xFFFF_FFFFL + }.getOrNull() ?: break + reportStartedIfPlaybackAdvanced() + if (playedFrames >= targetFrames) { + drained = true + break + } + Thread.sleep(5) + } + complete = drained + } + } catch (_: Exception) { + complete = false + } + mainHandler.post { + if (generation.get() == currentGeneration && track === nextTrack) { + releaseTrack(nextTrack) + track = null + abandonFocus() + channel.invokeMethod( + if (complete) "completed" else "error", + null, + ) + } + } + } + } + + private fun stopAndNotify(event: String) { + mainHandler.post { stop(notify = event) } + } + + private fun stop(notify: String?) { + if (track == null && !hasFocus) return + generation.incrementAndGet() + track?.let(::releaseTrack) + track = null + abandonFocus() + if (notify != null) channel.invokeMethod(notify, null) + } + + private fun requestFocus(): Boolean { + val result = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val request = + AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK) + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_ASSISTANCE_ACCESSIBILITY) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build(), + ) + .setOnAudioFocusChangeListener(focusListener, mainHandler) + .build() + focusRequest = request + audioManager.requestAudioFocus(request) + } else { + @Suppress("DEPRECATION") + audioManager.requestAudioFocus( + focusListener, + AudioManager.STREAM_MUSIC, + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK, + ) + } + hasFocus = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + return hasFocus + } + + private fun abandonFocus() { + if (!hasFocus) return + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + focusRequest?.let(audioManager::abandonAudioFocusRequest) + } else { + @Suppress("DEPRECATION") + audioManager.abandonAudioFocus(focusListener) + } + focusRequest = null + hasFocus = false + } + + private fun releaseTrack(audioTrack: AudioTrack) { + runCatching { audioTrack.pause() } + runCatching { audioTrack.flush() } + runCatching { audioTrack.stop() } + runCatching { audioTrack.release() } + } + + private companion object { + const val CHANNEL = "buzz/voice_audio" + } +} diff --git a/mobile/android/app/src/main/res/xml/backup_rules.xml b/mobile/android/app/src/main/res/xml/backup_rules.xml new file mode 100644 index 0000000000..188f46cf7a --- /dev/null +++ b/mobile/android/app/src/main/res/xml/backup_rules.xml @@ -0,0 +1,4 @@ + + + + diff --git a/mobile/android/app/src/main/res/xml/data_extraction_rules.xml b/mobile/android/app/src/main/res/xml/data_extraction_rules.xml new file mode 100644 index 0000000000..263cda4139 --- /dev/null +++ b/mobile/android/app/src/main/res/xml/data_extraction_rules.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/mobile/android/app/src/test/kotlin/xyz/block/buzz/mobile/VoiceAudioOutputTest.kt b/mobile/android/app/src/test/kotlin/xyz/block/buzz/mobile/VoiceAudioOutputTest.kt new file mode 100644 index 0000000000..f35963e01b --- /dev/null +++ b/mobile/android/app/src/test/kotlin/xyz/block/buzz/mobile/VoiceAudioOutputTest.kt @@ -0,0 +1,28 @@ +package xyz.block.buzz.mobile + +import android.media.AudioManager +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class VoiceAudioOutputTest { + @Test + fun `all focus losses stop Pocket voice`() { + assertTrue(shouldStopPocketVoiceForAudioFocusChange(AudioManager.AUDIOFOCUS_LOSS)) + assertTrue( + shouldStopPocketVoiceForAudioFocusChange( + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT, + ), + ) + assertTrue( + shouldStopPocketVoiceForAudioFocusChange( + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK, + ), + ) + } + + @Test + fun `focus gain does not interrupt Pocket voice`() { + assertFalse(shouldStopPocketVoiceForAudioFocusChange(AudioManager.AUDIOFOCUS_GAIN)) + } +} diff --git a/mobile/integration_test/pocket_voice_integration_test.dart b/mobile/integration_test/pocket_voice_integration_test.dart new file mode 100644 index 0000000000..7f17460402 --- /dev/null +++ b/mobile/integration_test/pocket_voice_integration_test.dart @@ -0,0 +1,154 @@ +import 'dart:io'; + +import 'package:buzz/shared/voice/pocket_model_downloader.dart'; +import 'package:buzz/shared/voice/pocket_voice_worker.dart'; +import 'package:buzz/shared/voice/voice_audio_output.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + testWidgets( + 'Pocket worker synthesizes valid PCM and cancels promptly', + (tester) async { + const configuredModelPath = String.fromEnvironment( + 'BUZZ_POCKET_MODEL_DIR', + ); + const downloadModel = bool.fromEnvironment('BUZZ_POCKET_DOWNLOAD'); + var modelPath = configuredModelPath; + Duration? downloadTime; + if (modelPath.isEmpty && downloadModel) { + final clock = Stopwatch()..start(); + final downloader = PocketModelDownloader(); + final directory = await downloader.install((downloaded, total, file) { + if (downloaded == total) { + debugPrint('Pocket model download complete: $file ($total bytes)'); + } + }); + clock.stop(); + downloadTime = clock.elapsed; + modelPath = directory.path; + expect(await downloader.verify(directory, hashContents: true), isTrue); + } + if (modelPath.isEmpty) { + fail( + 'Pass --dart-define=BUZZ_POCKET_MODEL_DIR= or ' + '--dart-define=BUZZ_POCKET_DOWNLOAD=true.', + ); + } + + final worker = PocketVoiceWorker(); + await worker.start(modelPath); + addTearDown(worker.dispose); + + final firstClock = Stopwatch()..start(); + final firstPlaybackClock = Stopwatch()..start(); + worker.synthesize(1, 'Pocket voice is running on this mobile device.'); + final first = await worker.responses + .where((response) => response is PocketWorkerAudio) + .cast() + .first + .timeout(const Duration(seconds: 30)); + firstClock.stop(); + final bytes = first.data.materialize().asUint8List(); + _expectValidPcm(bytes, first.sampleRate); + final audioSeconds = bytes.length / 2 / first.sampleRate; + final rtf = + first.synthesisTime.inMicroseconds / + Duration.microsecondsPerSecond / + audioSeconds; + final output = PlatformVoiceAudioOutput(); + final playbackStarted = output.events.firstWhere( + (event) => event == VoiceAudioEvent.started, + ); + final playbackCompleted = output.events.firstWhere( + (event) => event == VoiceAudioEvent.completed, + ); + final playbackClock = Stopwatch()..start(); + await output.play(bytes, first.sampleRate); + await playbackStarted.timeout(const Duration(seconds: 30)); + firstPlaybackClock.stop(); + await playbackCompleted.timeout(const Duration(seconds: 30)); + playbackClock.stop(); + expect( + playbackClock.elapsedMilliseconds, + greaterThanOrEqualTo((audioSeconds * 800).floor()), + ); + + final warmClock = Stopwatch()..start(); + worker.synthesize(2, 'The engine stays warm for the next response.'); + final warm = await worker.responses + .where((response) => response is PocketWorkerAudio) + .cast() + .where((audio) => audio.generation == 2) + .first + .timeout(const Duration(seconds: 30)); + warmClock.stop(); + _expectValidPcm(warm.data.materialize().asUint8List(), warm.sampleRate); + + final queuedCancelClock = Stopwatch()..start(); + worker.synthesize( + 3, + List.filled( + 20, + 'This queued request must remain cancelled before synthesis begins.', + ).join(' '), + ); + worker.cancel(); + final queuedCancelled = await worker.responses + .where((response) => response is PocketWorkerFailure) + .cast() + .where((failure) => failure.generation == 3) + .first + .timeout(const Duration(seconds: 10)); + queuedCancelClock.stop(); + expect(queuedCancelled.kind, PocketWorkerFailureKind.cancelled); + + final cancelClock = Stopwatch()..start(); + worker.synthesize( + 4, + List.filled( + 20, + 'This deliberately long sentence gives cancellation time to interrupt.', + ).join(' '), + ); + await Future.delayed(const Duration(milliseconds: 50)); + worker.cancel(); + final cancelled = await worker.responses + .where((response) => response is PocketWorkerFailure) + .cast() + .where((failure) => failure.generation == 4) + .first + .timeout(const Duration(seconds: 10)); + cancelClock.stop(); + expect(cancelled.kind, PocketWorkerFailureKind.cancelled); + + // Emitted in a stable shape so simulator/device runs are easy to compare. + debugPrint( + 'BUZZ_POCKET_METRICS ' + 'first_pcm_ms=${firstClock.elapsedMilliseconds} ' + 'ttfap_ms=${firstPlaybackClock.elapsedMilliseconds} ' + 'synthesis_ms=${first.synthesisTime.inMilliseconds} ' + 'audio_s=${audioSeconds.toStringAsFixed(3)} ' + 'rtf=${rtf.toStringAsFixed(3)} ' + 'playback_ms=${playbackClock.elapsedMilliseconds} ' + 'warm_pcm_ms=${warmClock.elapsedMilliseconds} ' + 'queued_cancel_ms=${queuedCancelClock.elapsedMilliseconds} ' + 'cancel_ms=${cancelClock.elapsedMilliseconds} ' + 'download_ms=${downloadTime?.inMilliseconds ?? 0} ' + 'rss_bytes=${ProcessInfo.currentRss} ' + 'peak_rss_bytes=${ProcessInfo.maxRss}', + ); + }, + timeout: const Timeout(Duration(minutes: 20)), + ); +} + +void _expectValidPcm(Uint8List bytes, int sampleRate) { + expect(sampleRate, 24000); + expect(bytes.length, greaterThan(4800)); + expect(bytes.length % 2, 0); + expect(bytes.any((byte) => byte != 0), isTrue); +} diff --git a/mobile/ios/Flutter/Debug.xcconfig b/mobile/ios/Flutter/Debug.xcconfig index 1f35905051..7ea6b87f99 100644 --- a/mobile/ios/Flutter/Debug.xcconfig +++ b/mobile/ios/Flutter/Debug.xcconfig @@ -19,3 +19,4 @@ APP_DISPLAY_NAME = Buzz // device signing beats the worktree default while unset variables still // fall through to the worktree values. #include? "AppOverrides.xcconfig" +#include "Voice.xcconfig" diff --git a/mobile/ios/Flutter/Release.xcconfig b/mobile/ios/Flutter/Release.xcconfig index d287c5fb43..e16556116c 100644 --- a/mobile/ios/Flutter/Release.xcconfig +++ b/mobile/ios/Flutter/Release.xcconfig @@ -9,3 +9,4 @@ APP_DISPLAY_NAME = Buzz CODE_SIGN_STYLE = Automatic CODE_SIGN_IDENTITY = iPhone Developer #include? "AppOverrides.xcconfig" +#include "Voice.xcconfig" diff --git a/mobile/ios/Flutter/Voice.xcconfig b/mobile/ios/Flutter/Voice.xcconfig new file mode 100644 index 0000000000..ed4fd00bbf --- /dev/null +++ b/mobile/ios/Flutter/Voice.xcconfig @@ -0,0 +1,2 @@ +BUZZ_VOICE_LIB_DIR = $(SRCROOT)/../.generated/voice/ios/$(PLATFORM_NAME)-$(CURRENT_ARCH) +OTHER_LDFLAGS = $(inherited) -Wl,-force_load,$(BUZZ_VOICE_LIB_DIR)/libbuzz_voice_mobile.a -lc++ -framework Accelerate -framework CoreML -framework Foundation -framework Metal -framework QuartzCore -framework Security diff --git a/mobile/ios/Podfile.lock b/mobile/ios/Podfile.lock index ba8c828d25..8ef612f5ff 100644 --- a/mobile/ios/Podfile.lock +++ b/mobile/ios/Podfile.lock @@ -15,6 +15,8 @@ PODS: - FlutterMacOS - image_picker_ios (0.0.1): - Flutter + - integration_test (0.0.1): + - Flutter - mobile_scanner (7.0.0): - Flutter - FlutterMacOS @@ -40,6 +42,7 @@ DEPENDENCIES: - Flutter (from `Flutter`) - flutter_secure_storage_darwin (from `.symlinks/plugins/flutter_secure_storage_darwin/darwin`) - image_picker_ios (from `.symlinks/plugins/image_picker_ios/ios`) + - integration_test (from `.symlinks/plugins/integration_test/ios`) - mobile_scanner (from `.symlinks/plugins/mobile_scanner/darwin`) - open_filex (from `.symlinks/plugins/open_filex/ios`) - package_info_plus (from `.symlinks/plugins/package_info_plus/ios`) @@ -64,6 +67,8 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/flutter_secure_storage_darwin/darwin" image_picker_ios: :path: ".symlinks/plugins/image_picker_ios/ios" + integration_test: + :path: ".symlinks/plugins/integration_test/ios" mobile_scanner: :path: ".symlinks/plugins/mobile_scanner/darwin" open_filex: @@ -86,6 +91,7 @@ SPEC CHECKSUMS: Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467 flutter_secure_storage_darwin: acdb3f316ed05a3e68f856e0353b133eec373a23 image_picker_ios: e0ece4aa2a75771a7de3fa735d26d90817041326 + integration_test: 4a889634ef21a45d28d50d622cf412dc6d9f586e mobile_scanner: 9157936403f5a0644ca3779a38ff8404c5434a93 open_filex: 432f3cd11432da3e39f47fcc0df2b1603854eff1 package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499 diff --git a/mobile/ios/Runner.xcodeproj/project.pbxproj b/mobile/ios/Runner.xcodeproj/project.pbxproj index 46c61ce2fc..8a92f373dc 100644 --- a/mobile/ios/Runner.xcodeproj/project.pbxproj +++ b/mobile/ios/Runner.xcodeproj/project.pbxproj @@ -17,6 +17,7 @@ 42C129326CE4E1B8E617B9CD /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = CD6B899582D0416ADBD8A68F /* Pods_RunnerTests.framework */; }; 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 74858FAE1ED2DC5600515810 /* AppDelegate.swift */; }; 7884E8682EC3CC0700C636F2 /* SceneDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7884E8672EC3CC0400C636F2 /* SceneDelegate.swift */; }; + A70D8C8D31879E5200B02246 /* VoiceAudioOutput.swift in Sources */ = {isa = PBXBuildFile; fileRef = A70D8C8C31879E5200B02246 /* VoiceAudioOutput.swift */; }; 97C146FC1CF9000F007C117D /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FA1CF9000F007C117D /* Main.storyboard */; }; 97C146FE1CF9000F007C117D /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FD1CF9000F007C117D /* Assets.xcassets */; }; 97C147011CF9000F007C117D /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */; }; @@ -72,6 +73,7 @@ 97C146FD1CF9000F007C117D /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 97C147001CF9000F007C117D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 97C147021CF9000F007C117D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + A70D8C8C31879E5200B02246 /* VoiceAudioOutput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceAudioOutput.swift; sourceTree = ""; }; CD6B899582D0416ADBD8A68F /* Pods_RunnerTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_RunnerTests.framework; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ @@ -171,6 +173,7 @@ 74858FAE1ED2DC5600515810 /* AppDelegate.swift */, 331C809A294A618700263BE5 /* MediaSanitizer.swift */, 7884E8672EC3CC0400C636F2 /* SceneDelegate.swift */, + A70D8C8C31879E5200B02246 /* VoiceAudioOutput.swift */, 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */, ); path = Runner; @@ -213,6 +216,7 @@ buildPhases = ( 1D327B920BBA7EF4545E2A92 /* [CP] Check Pods Manifest.lock */, 9740EEB61CF901F6004384FC /* Run Script */, + A70D8C8E31879E5200B02246 /* Build Pocket Voice */, 97C146EA1CF9000F007C117D /* Sources */, 97C146EB1CF9000F007C117D /* Frameworks */, 97C146EC1CF9000F007C117D /* Resources */, @@ -367,6 +371,22 @@ shellPath = /bin/sh; shellScript = "/bin/sh \"$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh\" build"; }; + A70D8C8E31879E5200B02246 /* Build Pocket Voice */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + ); + name = "Build Pocket Voice"; + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/bash; + shellScript = "cd \"$SRCROOT/../..\"\n. ./bin/activate-hermit\n./scripts/mobile-voice-native.sh build-ios-current\n"; + showEnvVarsInLog = 0; + }; E0B5862D106D142B580309AF /* [CP] Embed Pods Frameworks */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; @@ -403,6 +423,7 @@ 331C809B294A63AB00263BE5 /* MediaSanitizer.swift in Sources */, 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */, 7884E8682EC3CC0700C636F2 /* SceneDelegate.swift in Sources */, + A70D8C8D31879E5200B02246 /* VoiceAudioOutput.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/mobile/ios/Runner/AppDelegate.swift b/mobile/ios/Runner/AppDelegate.swift index 5edb4b6aaf..47e3b06c5f 100644 --- a/mobile/ios/Runner/AppDelegate.swift +++ b/mobile/ios/Runner/AppDelegate.swift @@ -7,6 +7,7 @@ import UserNotifications @objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate { private var mediaUploadChannel: FlutterMethodChannel? private var qrScannerChannel: FlutterMethodChannel? + private var voiceAudioOutput: VoiceAudioOutput? override func application( _ application: UIApplication, @@ -32,6 +33,9 @@ import UserNotifications qrScannerChannel?.setMethodCallHandler { call, result in Self.handleQrScannerMethodCall(call, result: result) } + voiceAudioOutput = VoiceAudioOutput( + messenger: engineBridge.applicationRegistrar.messenger() + ) } private static func handleQrScannerMethodCall( diff --git a/mobile/ios/Runner/Runner-Bridging-Header.h b/mobile/ios/Runner/Runner-Bridging-Header.h index 308a2a560b..b421e7bbb2 100644 --- a/mobile/ios/Runner/Runner-Bridging-Header.h +++ b/mobile/ios/Runner/Runner-Bridging-Header.h @@ -1 +1,2 @@ #import "GeneratedPluginRegistrant.h" +#import "../../../crates/buzz-voice-mobile/include/buzz_voice_mobile.h" diff --git a/mobile/ios/Runner/VoiceAudioOutput.swift b/mobile/ios/Runner/VoiceAudioOutput.swift new file mode 100644 index 0000000000..3340c0d4dd --- /dev/null +++ b/mobile/ios/Runner/VoiceAudioOutput.swift @@ -0,0 +1,208 @@ +import AVFoundation +import Flutter +import UIKit + +final class VoiceAudioOutput: NSObject, AVAudioPlayerDelegate { + private let channel: FlutterMethodChannel + private var player: AVAudioPlayer? + private var observers: [NSObjectProtocol] = [] + + init(messenger: FlutterBinaryMessenger) { + channel = FlutterMethodChannel( + name: "buzz/voice_audio", + binaryMessenger: messenger + ) + super.init() + channel.setMethodCallHandler { [weak self] call, result in + self?.handle(call, result: result) + } + observeAudioLifecycle() + } + + deinit { + for observer in observers { + NotificationCenter.default.removeObserver(observer) + } + } + + private func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "play": + guard + let arguments = call.arguments as? [String: Any], + let typedData = arguments["pcm"] as? FlutterStandardTypedData, + let sampleRate = arguments["sampleRate"] as? Int + else { + result(FlutterError(code: "invalid_arguments", message: "Expected PCM and sample rate.", details: nil)) + return + } + do { + try play(pcm: typedData.data, sampleRate: sampleRate) + result(nil) + } catch { + result(FlutterError(code: "playback_failed", message: error.localizedDescription, details: nil)) + } + case "stop": + stop(deactivate: true) + result(nil) + case "availableCapacity": + guard let path = call.arguments as? String else { + result(FlutterError(code: "invalid_arguments", message: "Expected a storage path.", details: nil)) + return + } + do { + let values = try URL(fileURLWithPath: path).resourceValues( + forKeys: [.volumeAvailableCapacityForImportantUsageKey] + ) + result(values.volumeAvailableCapacityForImportantUsage ?? 0) + } catch { + result(FlutterError(code: "storage_failed", message: error.localizedDescription, details: nil)) + } + case "excludeFromBackup": + guard let path = call.arguments as? String else { + result(FlutterError(code: "invalid_arguments", message: "Expected a storage path.", details: nil)) + return + } + do { + var url = URL(fileURLWithPath: path) + var values = URLResourceValues() + values.isExcludedFromBackup = true + try url.setResourceValues(values) + result(nil) + } catch { + result(FlutterError(code: "storage_failed", message: error.localizedDescription, details: nil)) + } + default: + result(FlutterMethodNotImplemented) + } + } + + private func play(pcm: Data, sampleRate: Int) throws { + stop(deactivate: false) + let session = AVAudioSession.sharedInstance() + try Self.configureAudioSession(session) + try session.setActive(true) + do { + let nextPlayer = try AVAudioPlayer(data: Self.waveData(pcm: pcm, sampleRate: sampleRate)) + nextPlayer.delegate = self + nextPlayer.prepareToPlay() + guard nextPlayer.play() else { + throw NSError( + domain: "BuzzVoice", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Unable to start voice playback."] + ) + } + player = nextPlayer + channel.invokeMethod("started", arguments: nil) + } catch { + try? session.setActive(false, options: [.notifyOthersOnDeactivation]) + throw error + } + } + + static func configureAudioSession(_ session: AVAudioSession) throws { + try session.setCategory(.playback, mode: .spokenAudio, options: [.duckOthers]) + } + + private func stop(deactivate: Bool) { + player?.stop() + player = nil + if deactivate { + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + } + } + + func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { + guard self.player === player else { return } + self.player = nil + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + channel.invokeMethod(flag ? "completed" : "error", arguments: nil) + } + + func audioPlayerDecodeErrorDidOccur(_ player: AVAudioPlayer, error: Error?) { + guard self.player === player else { return } + self.player = nil + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + channel.invokeMethod("error", arguments: error?.localizedDescription) + } + + private func observeAudioLifecycle() { + let center = NotificationCenter.default + observers.append( + center.addObserver( + forName: AVAudioSession.interruptionNotification, + object: nil, + queue: .main + ) { [weak self] notification in + guard + let rawType = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt, + AVAudioSession.InterruptionType(rawValue: rawType) == .began + else { return } + self?.interrupt(with: "interrupted") + } + ) + observers.append( + center.addObserver( + forName: AVAudioSession.routeChangeNotification, + object: nil, + queue: .main + ) { [weak self] notification in + guard + let rawReason = notification.userInfo?[AVAudioSessionRouteChangeReasonKey] as? UInt, + AVAudioSession.RouteChangeReason(rawValue: rawReason) == .oldDeviceUnavailable + else { return } + self?.interrupt(with: "routeLost") + } + ) + observers.append( + center.addObserver( + forName: UIApplication.didEnterBackgroundNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.interrupt(with: "backgrounded", requirePlayback: false) + } + ) + } + + private func interrupt(with event: String, requirePlayback: Bool = true) { + if requirePlayback && player == nil { return } + stop(deactivate: true) + channel.invokeMethod(event, arguments: nil) + } + + private static func waveData(pcm: Data, sampleRate: Int) -> Data { + var output = Data() + output.append(contentsOf: "RIFF".utf8) + output.appendLittleEndian(UInt32(pcm.count + 36)) + output.append(contentsOf: "WAVEfmt ".utf8) + output.appendLittleEndian(UInt32(16)) + output.appendLittleEndian(UInt16(1)) + output.appendLittleEndian(UInt16(1)) + output.appendLittleEndian(UInt32(sampleRate)) + output.appendLittleEndian(UInt32(sampleRate * 2)) + output.appendLittleEndian(UInt16(2)) + output.appendLittleEndian(UInt16(16)) + output.append(contentsOf: "data".utf8) + output.appendLittleEndian(UInt32(pcm.count)) + output.append(pcm) + return output + } +} + +private extension Data { + mutating func appendLittleEndian(_ value: T) { + var littleEndian = value.littleEndian + Swift.withUnsafeBytes(of: &littleEndian) { append(contentsOf: $0) } + } +} diff --git a/mobile/ios/RunnerTests/RunnerTests.swift b/mobile/ios/RunnerTests/RunnerTests.swift index a174c78684..d8fd610bf6 100644 --- a/mobile/ios/RunnerTests/RunnerTests.swift +++ b/mobile/ios/RunnerTests/RunnerTests.swift @@ -1,4 +1,6 @@ import Flutter +import Darwin +import AVFoundation import UIKit import XCTest @@ -6,6 +8,67 @@ import XCTest class RunnerTests: XCTestCase { + func testPocketVoiceUsesSpokenPlaybackAudioSession() throws { + let session = AVAudioSession.sharedInstance() + try VoiceAudioOutput.configureAudioSession(session) + XCTAssertEqual(session.category, .playback) + XCTAssertEqual(session.mode, .spokenAudio) + XCTAssertTrue(session.categoryOptions.contains(.duckOthers)) + } + + func testPocketVoiceProducesValidPcmWhenModelIsAvailable() throws { + let modelPath = + ProcessInfo.processInfo.environment["BUZZ_POCKET_MODEL_DIR"] + ?? UserDefaults.standard.string(forKey: "BuzzPocketModelTestPath") + guard let modelPath, FileManager.default.fileExists(atPath: modelPath) else { + throw XCTSkip("Set BUZZ_POCKET_MODEL_DIR to run the Pocket TTS integration test.") + } + + let loadStart = ContinuousClock.now + let engineResult = modelPath.withCString { + buzz_voice_engine_create($0, UInt8(BUZZ_VOICE_PRECISION_FP32)) + } + let loadTime = loadStart.duration(to: .now) + if let error = engineResult.error { + defer { buzz_voice_string_free(error) } + XCTFail(String(cString: error)) + return + } + let engine = try XCTUnwrap(engineResult.engine) + defer { buzz_voice_engine_destroy(engine) } + + let prompt = "Pocket voice is running on this iOS simulator." + let firstStart = ContinuousClock.now + let first = prompt.withCString { buzz_voice_engine_synthesize(engine, $0) } + let firstTime = firstStart.duration(to: .now) + let firstData = try validatedPcmData(first) + buzz_voice_pcm_free(first) + + let warmStart = ContinuousClock.now + let warm = prompt.withCString { buzz_voice_engine_synthesize(engine, $0) } + let warmTime = warmStart.duration(to: .now) + let warmData = try validatedPcmData(warm) + buzz_voice_pcm_free(warm) + + var usage = rusage() + getrusage(RUSAGE_SELF, &usage) + let audioSeconds = Double(firstData.count) / 2 / 24_000 + let firstSeconds = durationSeconds(firstTime) + let metrics = + "BUZZ_POCKET_METRICS " + + "load_s=\(durationSeconds(loadTime)) " + + "first_pcm_s=\(firstSeconds) " + + "audio_s=\(audioSeconds) " + + "rtf=\(firstSeconds / audioSeconds) " + + "warm_pcm_s=\(durationSeconds(warmTime)) " + + "peak_rss_bytes=\(usage.ru_maxrss) " + + "repeat_pcm_equal=\(firstData == warmData)" + let attachment = XCTAttachment(string: metrics) + attachment.name = metrics + attachment.lifetime = .keepAlways + add(attachment) + } + func testDynamicIslandQrScannerRecognizesTallSafeAreas() { for safeAreaTopInset in [51, 59, 62] { XCTAssertTrue( @@ -226,6 +289,27 @@ class RunnerTests: XCTestCase { Bundle(for: RunnerTests.self).url(forResource: name, withExtension: fileExtension)) return try Data(contentsOf: url) } + + private func validatedPcmData(_ pcm: BuzzVoicePcm) throws -> Data { + if let error = pcm.error { + defer { buzz_voice_string_free(error) } + XCTFail(String(cString: error)) + return Data() + } + XCTAssertEqual(pcm.sample_rate, 24_000) + XCTAssertGreaterThan(pcm.len, 2_400) + let samples = try XCTUnwrap(pcm.samples) + let buffer = UnsafeBufferPointer(start: samples, count: pcm.len) + XCTAssertTrue(buffer.contains { $0 != 0 }) + XCTAssertGreaterThan(buffer.map { abs(Int32($0)) }.max() ?? 0, 100) + return Data(bytes: samples, count: pcm.len * MemoryLayout.size) + } + + private func durationSeconds(_ duration: Duration) -> Double { + let components = duration.components + return Double(components.seconds) + + Double(components.attoseconds) / 1_000_000_000_000_000_000 + } } private enum RelayImagePolicyError: Error { diff --git a/mobile/lib/features/channels/channel_detail_page.dart b/mobile/lib/features/channels/channel_detail_page.dart index f94226ac80..36a8631df1 100644 --- a/mobile/lib/features/channels/channel_detail_page.dart +++ b/mobile/lib/features/channels/channel_detail_page.dart @@ -13,6 +13,7 @@ import '../../shared/widgets/avatar_image.dart'; import '../../shared/widgets/frosted_app_bar.dart'; import '../../shared/widgets/frosted_scaffold.dart'; import '../../shared/widgets/skeleton.dart'; +import '../../shared/voice/pocket_voice_controller.dart'; import '../profile/presence_cache_provider.dart'; import '../profile/profile_provider.dart'; import '../profile/user_cache_provider.dart'; @@ -35,6 +36,8 @@ import 'members_sheet.dart'; import 'message_actions.dart'; import 'message_content.dart'; import 'mentions/mention_candidates_provider.dart'; +import 'pocket_voice_button.dart'; +import 'pocket_voice_conversation.dart'; import 'read_state/deferred_read_state_update.dart'; import 'read_state/read_state_provider.dart'; import 'read_state/read_state_time.dart'; @@ -123,12 +126,32 @@ class ChannelDetailPage extends HookConsumerWidget { final detailsAsync = ref.watch(channelDetailsProvider(channel.id)); final channelsAsync = ref.watch(channelsProvider); final messagesState = ref.watch(channelMessagesProvider(channel.id)); + final membersState = ref.watch(channelMembersProvider(channel.id)); + final voiceConversation = useMemoized(PocketVoiceConversation.new); final sessionStatus = ref.watch(relaySessionProvider).status; final readState = ref.watch(readStateProvider); final currentPubkey = ref .watch(profileProvider) .whenData((value) => value?.pubkey) .value; + final voiceEventKey = + messagesState.value?.map((event) => event.id).join(',') ?? ''; + useEffect(() { + if (channel.isForum) return null; + final events = messagesState.value; + if (events == null) return null; + final spoken = voiceConversation.update( + events: events, + members: membersState.value, + currentPubkey: currentPubkey, + ); + for (final event in spoken) { + ref + .read(pocketVoiceProvider.notifier) + .speak('channel:${channel.id}', event.content); + } + return null; + }, [voiceEventKey, membersState.value, currentPubkey, channel.id]); // Only show channel-level typing (exclude thread-scoped entries and self). final typingEntries = ref .watch(channelTypingProvider(channel.id)) @@ -254,6 +277,8 @@ class ChannelDetailPage extends HookConsumerWidget { ], ), actions: [ + if (!resolvedChannel.isForum) + PocketVoiceButton(conversationKey: 'channel:${channel.id}'), _MembersButton( channelId: resolvedChannel.id, channel: resolvedChannel, diff --git a/mobile/lib/features/channels/pocket_voice_button.dart b/mobile/lib/features/channels/pocket_voice_button.dart new file mode 100644 index 0000000000..dc8571cbe1 --- /dev/null +++ b/mobile/lib/features/channels/pocket_voice_button.dart @@ -0,0 +1,56 @@ +import 'package:flutter/material.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:lucide_icons_flutter/lucide_icons.dart'; + +import '../../shared/theme/theme.dart'; +import '../../shared/voice/pocket_model_provider.dart'; +import '../../shared/voice/pocket_voice_controller.dart'; + +class PocketVoiceButton extends ConsumerWidget { + const PocketVoiceButton({super.key, required this.conversationKey}); + + final String conversationKey; + + @override + Widget build(BuildContext context, WidgetRef ref) { + final voice = ref.watch(pocketVoiceProvider); + final model = ref.watch(pocketModelProvider); + final active = voice.enabled && voice.conversationKey == conversationKey; + + return IconButton( + color: active ? context.colors.primary : null, + tooltip: active ? 'Stop Pocket voice' : 'Start Pocket voice', + onPressed: () async { + if (active) { + await ref.read(pocketVoiceProvider.notifier).disable(); + return; + } + if (model.phase != PocketModelPhase.ready) { + if (context.mounted) { + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar( + content: Text('Download Pocket voice in Settings first.'), + ), + ); + } + return; + } + try { + await ref.read(pocketVoiceProvider.notifier).enable(conversationKey); + } catch (error) { + if (context.mounted) { + ScaffoldMessenger.of( + context, + ).showSnackBar(SnackBar(content: Text(error.toString()))); + } + } + }, + icon: voice.phase == PocketVoicePhase.loading && active + ? const SizedBox.square( + dimension: 18, + child: CircularProgressIndicator(strokeWidth: 2), + ) + : const Icon(LucideIcons.audioLines, size: 21), + ); + } +} diff --git a/mobile/lib/features/channels/pocket_voice_conversation.dart b/mobile/lib/features/channels/pocket_voice_conversation.dart new file mode 100644 index 0000000000..bdde9fbe57 --- /dev/null +++ b/mobile/lib/features/channels/pocket_voice_conversation.dart @@ -0,0 +1,51 @@ +import '../../shared/relay/relay.dart'; +import 'channel_management_provider.dart'; + +/// Selects newly-arrived bot messages for Pocket voice. +/// +/// Membership is authoritative: messages are never spoken until the channel's +/// member list has resolved, and history present when the view opens is only +/// used to establish the deduplication baseline. +class PocketVoiceConversation { + final Set _seen = {}; + bool _initialized = false; + + List update({ + required List events, + required List? members, + required String? currentPubkey, + String? threadHeadId, + }) { + final unseen = []; + for (final event in events) { + if (_seen.add(event.id)) unseen.add(event); + } + if (!_initialized) { + _initialized = true; + return const []; + } + if (members == null) return const []; + + final bots = { + for (final member in members) + if (member.isBot) member.pubkey.toLowerCase(), + }; + final self = currentPubkey?.toLowerCase(); + return [ + for (final event in unseen) + if ((event.kind == EventKind.streamMessage || + event.kind == EventKind.streamMessageV2) && + event.pubkey.toLowerCase() != self && + bots.contains(event.pubkey.toLowerCase()) && + event.content.trim().length > 1 && + !event.content.trimLeft().startsWith('[System]') && + _belongsToConversation(event, threadHeadId)) + event, + ]; + } + + static bool _belongsToConversation(NostrEvent event, String? threadHeadId) { + final parentId = event.threadReference.parentId; + return threadHeadId == null ? parentId == null : parentId == threadHeadId; + } +} diff --git a/mobile/lib/features/channels/send_message_provider.dart b/mobile/lib/features/channels/send_message_provider.dart index 3659bda4bf..fa37d3cae8 100644 --- a/mobile/lib/features/channels/send_message_provider.dart +++ b/mobile/lib/features/channels/send_message_provider.dart @@ -1,6 +1,7 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; import '../../shared/relay/relay.dart'; +import '../../shared/voice/pocket_voice_controller.dart'; import '../channels/channel_management_provider.dart'; import '../profile/user_cache_provider.dart'; import '../profile/user_profile.dart'; @@ -15,6 +16,7 @@ class SendMessage { final void Function(String channelId, NostrEvent event) _addLocalMessage; final void Function(String channelId, String eventId) _completeLocalMessage; final void Function(String channelId, String eventId) _removeLocalMessage; + final Future Function()? _onBeforeSend; SendMessage({ required SignedEventRelay signedEventRelay, @@ -25,12 +27,14 @@ class SendMessage { required void Function(String channelId, String eventId) completeLocalMessage, required void Function(String channelId, String eventId) removeLocalMessage, + Future Function()? onBeforeSend, }) : _signedEventRelay = signedEventRelay, _fetchMembers = fetchMembers, _readUserCache = readUserCache, _addLocalMessage = addLocalMessage, _completeLocalMessage = completeLocalMessage, - _removeLocalMessage = removeLocalMessage; + _removeLocalMessage = removeLocalMessage, + _onBeforeSend = onBeforeSend; /// Send a text message to a channel. /// @@ -47,6 +51,7 @@ class SendMessage { List? mentionPubkeys, List> mediaTags = const [], }) async { + await _onBeforeSend?.call(); // Use explicitly passed pubkeys, or resolve @mentions against // channel members to avoid matching the wrong user. final resolvedMentions = @@ -179,5 +184,10 @@ final sendMessageProvider = Provider((ref) { removeLocalMessage: (channelId, eventId) => ref .read(channelMessagesProvider(channelId).notifier) .removeLocalMessage(eventId), + onBeforeSend: () async { + if (ref.read(pocketVoiceProvider).enabled) { + await ref.read(pocketVoiceProvider.notifier).interrupt(); + } + }, ); }); diff --git a/mobile/lib/features/channels/thread_detail_page.dart b/mobile/lib/features/channels/thread_detail_page.dart index d00973b4d1..2e1898e022 100644 --- a/mobile/lib/features/channels/thread_detail_page.dart +++ b/mobile/lib/features/channels/thread_detail_page.dart @@ -7,9 +7,11 @@ import '../../shared/theme/theme.dart'; import '../../shared/widgets/avatar_image.dart'; import '../../shared/widgets/frosted_app_bar.dart'; import '../../shared/widgets/frosted_scaffold.dart'; +import '../../shared/voice/pocket_voice_controller.dart'; import '../profile/user_cache_provider.dart'; import '../profile/user_profile.dart'; import 'channel_link_navigation.dart'; +import 'channel_management_provider.dart'; import 'channel_typing_provider.dart'; import 'thread_replies_provider.dart'; import 'channels_provider.dart'; @@ -20,6 +22,8 @@ import '../profile/user_profile_sheet.dart'; import 'message_actions.dart'; import 'message_content.dart'; import 'mentions/mention_candidates_provider.dart'; +import 'pocket_voice_button.dart'; +import 'pocket_voice_conversation.dart'; import 'reaction_row.dart'; import 'read_state/read_state_format.dart'; import 'read_state/read_state_provider.dart'; @@ -67,6 +71,35 @@ class ThreadDetailPage extends HookConsumerWidget { }); final fetchedReplies = replyMessages.value; + final membersState = ref.watch(channelMembersProvider(channelId)); + final voiceConversation = useMemoized(PocketVoiceConversation.new); + final voiceEventKey = + repliesState.value?.map((event) => event.id).join(',') ?? ''; + useEffect( + () { + final events = repliesState.value; + if (events == null) return null; + final spoken = voiceConversation.update( + events: events, + members: membersState.value, + currentPubkey: currentPubkey, + threadHeadId: threadHead.id, + ); + for (final event in spoken) { + ref + .read(pocketVoiceProvider.notifier) + .speak('thread:$channelId:${threadHead.id}', event.content); + } + return null; + }, + [ + voiceEventKey, + membersState.value, + currentPubkey, + channelId, + threadHead.id, + ], + ); final allMsgs = fetchedReplies == null ? allMessages : [ @@ -153,7 +186,14 @@ class ThreadDetailPage extends HookConsumerWidget { }); return FrostedScaffold( - appBar: const FrostedAppBar(title: Text('Thread')), + appBar: FrostedAppBar( + title: const Text('Thread'), + actions: [ + PocketVoiceButton( + conversationKey: 'thread:$channelId:${threadHead.id}', + ), + ], + ), body: Column( children: [ Expanded( diff --git a/mobile/lib/features/settings/settings_page.dart b/mobile/lib/features/settings/settings_page.dart index f089f63915..4ec6dbfb12 100644 --- a/mobile/lib/features/settings/settings_page.dart +++ b/mobile/lib/features/settings/settings_page.dart @@ -9,6 +9,8 @@ import '../../shared/auth/auth.dart'; import '../../shared/clipboard_utils.dart'; import '../../shared/relay/relay.dart'; import '../../shared/theme/theme.dart'; +import '../../shared/voice/pocket_model_manifest.dart'; +import '../../shared/voice/pocket_model_provider.dart'; import '../../shared/widgets/app_list.dart'; import '../../shared/widgets/app_list_card.dart'; import '../../shared/widgets/frosted_app_bar.dart'; @@ -18,6 +20,7 @@ import 'theme_picker_page.dart'; part 'settings_page/appearance_section.dart'; part 'settings_page/connection_section.dart'; +part 'settings_page/voice_section.dart'; class SettingsPage extends HookConsumerWidget { const SettingsPage({super.key, required this.profileHeader}); @@ -42,6 +45,7 @@ class SettingsPage extends HookConsumerWidget { children: [ profileHeader, const _AppearanceSection(), + const _VoiceSection(), const _ConnectionSection(), const _RemoveCommunitySection(), ], diff --git a/mobile/lib/features/settings/settings_page/voice_section.dart b/mobile/lib/features/settings/settings_page/voice_section.dart new file mode 100644 index 0000000000..6ac53448db --- /dev/null +++ b/mobile/lib/features/settings/settings_page/voice_section.dart @@ -0,0 +1,69 @@ +part of '../settings_page.dart'; + +class _VoiceSection extends ConsumerWidget { + const _VoiceSection(); + + @override + Widget build(BuildContext context, WidgetRef ref) { + final model = ref.watch(pocketModelProvider); + final downloading = model.phase == PocketModelPhase.downloading; + final status = switch (model.phase) { + PocketModelPhase.checking => 'Checking…', + PocketModelPhase.absent => 'Not downloaded', + PocketModelPhase.downloading => + '${(model.progress * 100).clamp(0, 100).round()}%', + PocketModelPhase.verifying => 'Verifying…', + PocketModelPhase.ready => 'Ready', + PocketModelPhase.cancelled => 'Cancelled', + PocketModelPhase.insufficientSpace => 'Needs more space', + PocketModelPhase.error => 'Download failed', + }; + final action = switch (model.phase) { + PocketModelPhase.ready || + PocketModelPhase.checking || + PocketModelPhase.verifying => null, + PocketModelPhase.downloading => + () => ref.read(pocketModelProvider.notifier).cancel(), + _ => () => ref.read(pocketModelProvider.notifier).download(), + }; + + return AppListCard( + label: 'Voice', + children: [ + AppListRow( + icon: LucideIcons.audioLines, + title: 'Pocket voice', + subtitle: + model.message ?? + 'Private, on-device speech. Downloads ' + '${pocketModelRuntimeBytes ~/ 1000000} MB after install.', + subtitleMaxLines: 3, + value: status, + trailing: downloading + ? SizedBox.square( + dimension: 20, + child: CircularProgressIndicator( + value: model.total == 0 ? null : model.progress, + strokeWidth: 2, + ), + ) + : action == null + ? null + : Icon( + downloading ? LucideIcons.x : LucideIcons.download, + size: 18, + color: context.colors.primary, + ), + onTap: action, + ), + const AppListRow( + icon: LucideIcons.hardDriveDownload, + title: 'Model storage', + value: '${pocketModelRuntimeBytes ~/ 1000000} MB', + subtitle: 'Stored in application support, not duplicated in assets.', + subtitleMaxLines: 2, + ), + ], + ); + } +} diff --git a/mobile/lib/shared/voice/pocket_model_downloader.dart b/mobile/lib/shared/voice/pocket_model_downloader.dart new file mode 100644 index 0000000000..38725bf273 --- /dev/null +++ b/mobile/lib/shared/voice/pocket_model_downloader.dart @@ -0,0 +1,390 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +import 'package:flutter/services.dart'; +import 'package:http/http.dart' as http; +import 'package:path_provider/path_provider.dart'; +import 'package:pointycastle/digests/sha256.dart'; +import 'package:uuid/uuid.dart'; + +import 'pocket_model_manifest.dart'; + +const _manifestName = '.buzz-model-manifest'; +const _legacyPocketModelVersion = '3'; +const _storageChannel = MethodChannel('buzz/voice_audio'); +const _minimumReserveBytes = 64 * 1024 * 1024; +const _defaultSendTimeout = Duration(seconds: 30); +const _defaultBodyIdleTimeout = Duration(seconds: 30); + +class PocketDownloadCancelled implements Exception { + const PocketDownloadCancelled(); +} + +class PocketInsufficientSpace implements Exception { + final int required; + final int available; + + const PocketInsufficientSpace(this.required, this.available); +} + +class PocketModelDownloadException implements Exception { + final String message; + final bool retryable; + + const PocketModelDownloadException(this.message, {this.retryable = false}); + + @override + String toString() => message; +} + +typedef PocketDownloadProgress = + void Function(int downloaded, int total, String currentFile); + +class PocketModelDownloader { + final http.Client Function() clientFactory; + final Future Function() applicationSupportDirectory; + final Future Function(String path) availableCapacity; + final Future Function(String path) excludeFromBackup; + final List artifacts; + final String installationId; + final Duration sendTimeout; + final Duration bodyIdleTimeout; + + http.Client? _client; + bool _cancelled = false; + + PocketModelDownloader({ + http.Client Function()? clientFactory, + Future Function()? applicationSupportDirectory, + Future Function(String path)? availableCapacity, + Future Function(String path)? excludeFromBackup, + this.artifacts = const [], + String? installationId, + this.sendTimeout = _defaultSendTimeout, + this.bodyIdleTimeout = _defaultBodyIdleTimeout, + }) : clientFactory = clientFactory ?? http.Client.new, + applicationSupportDirectory = + applicationSupportDirectory ?? getApplicationSupportDirectory, + availableCapacity = availableCapacity ?? _platformAvailableCapacity, + excludeFromBackup = excludeFromBackup ?? _platformExcludeFromBackup, + installationId = installationId ?? const Uuid().v4(); + + List get _artifacts => + artifacts.isEmpty ? pocketModelArtifacts : artifacts; + + int get _downloadBytes => + _artifacts.fold(0, (total, artifact) => total + artifact.size); + + Future modelDirectory() async { + final support = await applicationSupportDirectory(); + return Directory( + '${support.path}/buzz/models/pocket-tts/v$pocketModelVersion', + ); + } + + Future isReady() async { + final directory = await modelDirectory(); + final ready = await verify(directory, hashContents: false); + if (ready) { + await _cleanupLegacyVersion(directory.parent); + } + return ready; + } + + Future verify(Directory directory, {required bool hashContents}) async { + final manifest = File('${directory.path}/$_manifestName'); + if (!await manifest.isFile()) return false; + try { + final decoded = + jsonDecode(await manifest.readAsString()) as Map; + if (decoded['version'] != pocketModelVersion || + decoded['revision'] != pocketModelRevision || + decoded['precision'] != pocketModelPrecision || + decoded['complete'] != true) { + return false; + } + for (final artifact in _artifacts) { + final file = File('${directory.path}/${artifact.name}'); + if (!await file.isFile() || await file.length() != artifact.size) { + return false; + } + if (hashContents && await _sha256File(file) != artifact.sha256) { + return false; + } + } + return File('${directory.path}/MODEL_LICENSE.txt').isFile(); + } catch (_) { + return false; + } + } + + Future install(PocketDownloadProgress onProgress) async { + _cancelled = false; + final finalDirectory = await modelDirectory(); + final parent = finalDirectory.parent; + await parent.create(recursive: true); + await _cleanupAbandoned(parent, finalDirectory.path); + + final available = await availableCapacity(parent.path); + final reserve = max(_minimumReserveBytes, (_downloadBytes * 0.1).ceil()); + final required = _downloadBytes + reserve; + if (available < required) { + throw PocketInsufficientSpace(required, available); + } + + final staging = Directory( + '${parent.path}/.pocket-tts-v$pocketModelVersion.install-$installationId', + ); + await staging.create(recursive: true); + await excludeFromBackup(staging.path); + var downloaded = 0; + _client = clientFactory(); + try { + for (final artifact in _artifacts) { + _throwIfCancelled(); + await _ensureCapacity(parent.path, artifact.size); + await _downloadArtifact( + artifact, + staging, + (fileBytes) => + onProgress(downloaded + fileBytes, _downloadBytes, artifact.name), + ); + downloaded += artifact.size; + onProgress(downloaded, _downloadBytes, artifact.name); + } + + await File( + '${staging.path}/MODEL_LICENSE.txt', + ).writeAsString(pocketModelLicenseText, flush: true); + await File('${staging.path}/$_manifestName').writeAsString( + jsonEncode({ + 'version': pocketModelVersion, + 'revision': pocketModelRevision, + 'precision': pocketModelPrecision, + 'complete': true, + 'artifacts': [ + for (final artifact in _artifacts) + { + 'name': artifact.name, + 'size': artifact.size, + 'sha256': artifact.sha256, + }, + ], + }), + flush: true, + ); + if (!await verify(staging, hashContents: true)) { + throw const PocketModelDownloadException( + 'Pocket model verification failed after download.', + ); + } + + final backup = Directory( + '${parent.path}/.pocket-tts-v$pocketModelVersion.old-$installationId', + ); + if (await finalDirectory.exists()) { + await finalDirectory.rename(backup.path); + } + try { + await staging.rename(finalDirectory.path); + } catch (_) { + if (await backup.exists() && !await finalDirectory.exists()) { + await backup.rename(finalDirectory.path); + } + rethrow; + } + if (await backup.exists()) { + await backup.delete(recursive: true); + } + await excludeFromBackup(finalDirectory.path); + await _cleanupLegacyVersion(parent); + return finalDirectory; + } on PocketDownloadCancelled { + rethrow; + } finally { + _client?.close(); + _client = null; + if (await staging.exists()) { + await staging.delete(recursive: true); + } + } + } + + void cancel() { + _cancelled = true; + _client?.close(); + } + + Future _downloadArtifact( + PocketModelArtifact artifact, + Directory staging, + void Function(int fileBytes) onProgress, + ) async { + Object? lastError; + for (var attempt = 0; attempt < 3; attempt += 1) { + _throwIfCancelled(); + final part = File('${staging.path}/${artifact.name}.part'); + if (await part.exists()) await part.delete(); + try { + final request = http.Request('GET', artifact.url); + final response = await _client!.send(request).timeout(sendTimeout); + if (response.statusCode != HttpStatus.ok) { + final retryable = + response.statusCode == 408 || + response.statusCode == 429 || + response.statusCode >= 500; + throw PocketModelDownloadException( + '${artifact.name} download returned HTTP ' + '${response.statusCode}.', + retryable: retryable, + ); + } + if (response.contentLength case final length? + when length != artifact.size) { + throw PocketModelDownloadException( + '${artifact.name} expected ${artifact.size} bytes, ' + 'server reported $length.', + ); + } + + final digest = SHA256Digest(); + final sink = part.openWrite(); + var bytes = 0; + try { + await for (final chunk in response.stream.timeout(bodyIdleTimeout)) { + _throwIfCancelled(); + bytes += chunk.length; + if (bytes > artifact.size) { + throw PocketModelDownloadException( + '${artifact.name} exceeded ${artifact.size} bytes.', + ); + } + final typed = Uint8List.fromList(chunk); + digest.update(typed, 0, typed.length); + sink.add(chunk); + onProgress(bytes); + } + await sink.flush(); + } finally { + await sink.close(); + } + if (bytes != artifact.size) { + throw PocketModelDownloadException( + '${artifact.name} expected ${artifact.size} bytes, received $bytes.', + retryable: true, + ); + } + final output = Uint8List(digest.digestSize); + digest.doFinal(output, 0); + final hash = output + .map((byte) => byte.toRadixString(16).padLeft(2, '0')) + .join(); + if (hash != artifact.sha256) { + throw PocketModelDownloadException( + '${artifact.name} checksum did not match.', + ); + } + await part.rename('${staging.path}/${artifact.name}'); + return; + } on PocketDownloadCancelled { + rethrow; + } catch (error) { + _throwIfCancelled(); + lastError = error; + if (await part.exists()) await part.delete(); + final retryable = + error is SocketException || + error is TimeoutException || + (error is PocketModelDownloadException && error.retryable); + if (!retryable || attempt == 2) break; + if (error is TimeoutException || error is SocketException) { + _client?.close(); + _client = clientFactory(); + } + await Future.delayed(Duration(milliseconds: 250 << attempt)); + } + } + if (lastError is PocketModelDownloadException) throw lastError; + throw PocketModelDownloadException( + 'Unable to download ${artifact.name}: $lastError', + retryable: true, + ); + } + + Future _ensureCapacity(String path, int nextFileBytes) async { + final available = await availableCapacity(path); + final reserve = max(_minimumReserveBytes, (nextFileBytes * 0.1).ceil()); + if (available < nextFileBytes + reserve) { + throw PocketInsufficientSpace(nextFileBytes + reserve, available); + } + } + + Future _cleanupAbandoned(Directory parent, String finalPath) async { + if (!await parent.exists()) return; + final finalDirectory = Directory(finalPath); + final backups = []; + await for (final entity in parent.list()) { + if (entity.path == finalPath) continue; + final name = entity.uri.pathSegments + .where((segment) => segment.isNotEmpty) + .last; + if (entity is! Directory) continue; + if (name.startsWith('.pocket-tts-v$pocketModelVersion.old-')) { + backups.add(entity); + } else if (name.startsWith('.pocket-tts-v$pocketModelVersion.install-')) { + await entity.delete(recursive: true); + } + } + if (!await finalDirectory.exists() && backups.isNotEmpty) { + backups.sort( + (left, right) => + right.statSync().modified.compareTo(left.statSync().modified), + ); + await backups.removeAt(0).rename(finalPath); + } + for (final backup in backups) { + if (await backup.exists()) await backup.delete(recursive: true); + } + } + + Future _cleanupLegacyVersion(Directory parent) async { + final legacy = Directory('${parent.path}/v$_legacyPocketModelVersion'); + try { + if (await legacy.exists()) { + await legacy.delete(recursive: true); + } + } on FileSystemException { + // The verified v4 model remains usable. A later readiness check retries + // cleanup instead of turning a successful upgrade into an error state. + } + } + + void _throwIfCancelled() { + if (_cancelled) throw const PocketDownloadCancelled(); + } +} + +Future _sha256File(File file) async { + final digest = SHA256Digest(); + await for (final chunk in file.openRead()) { + final typed = Uint8List.fromList(chunk); + digest.update(typed, 0, typed.length); + } + final output = Uint8List(digest.digestSize); + digest.doFinal(output, 0); + return output.map((byte) => byte.toRadixString(16).padLeft(2, '0')).join(); +} + +Future _platformAvailableCapacity(String path) async => + await _storageChannel.invokeMethod('availableCapacity', path) ?? 0; + +Future _platformExcludeFromBackup(String path) => + _storageChannel.invokeMethod('excludeFromBackup', path); + +extension on File { + Future isFile() async => + await exists() && + await stat().then((stat) => stat.type == FileSystemEntityType.file); +} diff --git a/mobile/lib/shared/voice/pocket_model_manifest.dart b/mobile/lib/shared/voice/pocket_model_manifest.dart new file mode 100644 index 0000000000..d3676091b5 --- /dev/null +++ b/mobile/lib/shared/voice/pocket_model_manifest.dart @@ -0,0 +1,127 @@ +import 'package:flutter/foundation.dart'; + +@immutable +class PocketModelArtifact { + final String name; + final Uri url; + final int size; + final String sha256; + + const PocketModelArtifact({ + required this.name, + required this.url, + required this.size, + required this.sha256, + }); +} + +const pocketModelVersion = '4'; +const pocketModelRevision = '58a6d00cf13d239b6748cb0769f35c580a8f606c'; +const pocketModelPrecision = 'fp32'; +const pocketModelCoreBytes = 439555904; +const pocketModelRuntimeBytes = 440213643; +const _pocketModelBase = + 'https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/' + '$pocketModelRevision/onnx/english_2026-04'; +const _pocketLicenseUrl = + 'https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/' + '$pocketModelRevision/onnx/LICENSE'; +const _pocketVoiceUrl = + 'https://huggingface.co/kyutai/tts-voices/resolve/' + '323332d33f997de8394f24a193e1a76df720e01a/' + 'vctk/p333_023_enhanced.wav'; + +final pocketModelArtifacts = [ + _artifact( + 'bundle.json', + 24381, + 'bab643150f437f37df080a710520ff39ed9ebd9a339f8ebdc739f7eddfc28b3f', + ), + _artifact( + 'bos_before_voice.npy', + 4224, + 'f46edf4f7007b7ba4ea58831f49d003e59e167b4641c44bb3addfe9231a780b1', + ), + _artifact( + 'tokenizer.model', + 59339, + 'd461765ae179566678c93091c5fa6f2984c31bbe990bf1aa62d92c64d91bc3f6', + ), + _artifact( + 'flow_lm_main.onnx', + 302742149, + '6d18315e2c33ca3e3aa4a4e3dca22f56d007fd823127e24948b37695bf54190f', + ), + _artifact( + 'flow_lm_flow.onnx', + 39097095, + '085d239f68897e28fb06e95c743738ad8b8c092ee6dc55f5491313e81ff08062', + ), + _artifact( + 'mimi_decoder.onnx', + 41471926, + '86f038caa02a96a0ff9c25526a0ff43a4906c418197ed72d3e30f720ac7ce802', + ), + _artifact( + 'mimi_encoder.onnx', + 39768446, + '853e2ca623b8782d94c3745ec6133bfdff7ce33d9b11128bd29ea03f28d76e3d', + ), + _artifact( + 'text_conditioner.onnx', + 16388344, + '4ecee995fb69f85c7a7493d11f7b5ee15d9950facc7ab3f5c9c49ef1e03847bb', + ), + PocketModelArtifact( + name: 'reference_sample.wav', + url: Uri.parse(_pocketVoiceUrl), + size: 639084, + sha256: 'a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f', + ), + PocketModelArtifact( + name: 'LICENSE', + url: Uri.parse(_pocketLicenseUrl), + size: 18655, + sha256: 'fe7b4ce83b8381cc5b216bbb4af73c570688d1b819c73bbaed8ca401f4677cd6', + ), +]; + +PocketModelArtifact _artifact(String name, int size, String sha256) => + PocketModelArtifact( + name: name, + url: Uri.parse('$_pocketModelBase/$name'), + size: size, + sha256: sha256, + ); + +const pocketModelLicenseText = ''' +Pocket TTS +© Kyutai. + +Licensed under the Creative Commons Attribution 4.0 International License +(CC-BY-4.0). License text: https://creativecommons.org/licenses/by/4.0/ + +Original model by Kyutai: https://huggingface.co/kyutai/pocket-tts +Paper: Charles, Roebel, et al., Pocket TTS (arXiv:2509.06926). +Mimi neural codec by Kyutai is bundled as part of the model. + +April 2026 ONNX export by KevinAHM: +https://huggingface.co/KevinAHM/pocket-tts-onnx +Pinned revision: 58a6d00cf13d239b6748cb0769f35c580a8f606c +Bundle: english_2026-04, full-precision graphs. + +Bundled reference voice (reference_sample.wav): +"Mary (f, conversation)" preset from the Kyutai TTS demo voice catalogue +(https://kyutai.org/tts), distributed via +https://huggingface.co/kyutai/tts-voices as `vctk/p333_023_enhanced.wav`. +Original recording from the Voice Cloning Toolkit (VCTK) corpus, speaker p333: +https://datashare.ed.ac.uk/handle/10283/3443 (CC-BY-4.0). +Recording enhancement (denoise/dereverb) by ai-coustics: +https://ai-coustics.com/ + +Buzz ships all ONNX/model artifacts and the reference voice WAV unmodified, +renamed only by placement in the local model directory. + +Provided "AS IS", without warranty of any kind, express or implied. See the +license text for full warranty disclaimer. +'''; diff --git a/mobile/lib/shared/voice/pocket_model_provider.dart b/mobile/lib/shared/voice/pocket_model_provider.dart new file mode 100644 index 0000000000..98668d297d --- /dev/null +++ b/mobile/lib/shared/voice/pocket_model_provider.dart @@ -0,0 +1,117 @@ +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +import 'pocket_model_downloader.dart'; + +enum PocketModelPhase { + checking, + absent, + downloading, + verifying, + ready, + cancelled, + insufficientSpace, + error, +} + +@immutable +class PocketModelState { + final PocketModelPhase phase; + final int downloaded; + final int total; + final String? path; + final String? message; + + const PocketModelState({ + required this.phase, + this.downloaded = 0, + this.total = 0, + this.path, + this.message, + }); + + double get progress => total == 0 ? 0 : downloaded / total; +} + +final pocketModelDownloaderProvider = Provider( + (_) => PocketModelDownloader(), +); + +final pocketModelProvider = + NotifierProvider( + PocketModelNotifier.new, + ); + +class PocketModelNotifier extends Notifier { + PocketModelDownloader? _active; + + @override + PocketModelState build() { + ref.onDispose(() => _active?.cancel()); + Future.microtask(check); + return const PocketModelState(phase: PocketModelPhase.checking); + } + + Future check() async { + final downloader = ref.read(pocketModelDownloaderProvider); + if (await downloader.isReady()) { + final directory = await downloader.modelDirectory(); + state = PocketModelState( + phase: PocketModelPhase.ready, + path: directory.path, + ); + } else { + state = const PocketModelState(phase: PocketModelPhase.absent); + } + } + + Future download() async { + if (_active != null) return; + final downloader = ref.read(pocketModelDownloaderProvider); + _active = downloader; + try { + final directory = await downloader.install((downloaded, total, _) { + state = PocketModelState( + phase: PocketModelPhase.downloading, + downloaded: downloaded, + total: total, + ); + }); + state = const PocketModelState(phase: PocketModelPhase.verifying); + if (!await downloader.verify(directory, hashContents: false)) { + throw const PocketModelDownloadException( + 'Pocket model verification failed.', + ); + } + state = PocketModelState( + phase: PocketModelPhase.ready, + path: directory.path, + ); + } on PocketDownloadCancelled { + state = const PocketModelState(phase: PocketModelPhase.cancelled); + } on PocketInsufficientSpace { + state = PocketModelState( + phase: PocketModelPhase.insufficientSpace, + message: 'Not enough free space for Pocket voice.', + ); + } on FileSystemException catch (error) { + state = PocketModelState( + phase: PocketModelPhase.error, + message: error.osError?.errorCode == 28 + ? 'Pocket voice ran out of disk space.' + : 'Pocket voice storage failed: ${error.message}', + ); + } catch (error) { + state = PocketModelState( + phase: PocketModelPhase.error, + message: error.toString(), + ); + } finally { + _active = null; + } + } + + void cancel() => _active?.cancel(); +} diff --git a/mobile/lib/shared/voice/pocket_voice_controller.dart b/mobile/lib/shared/voice/pocket_voice_controller.dart new file mode 100644 index 0000000000..6fea633974 --- /dev/null +++ b/mobile/lib/shared/voice/pocket_voice_controller.dart @@ -0,0 +1,364 @@ +import 'dart:async'; +import 'dart:collection'; + +import 'package:flutter/foundation.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +import '../relay/relay.dart'; +import 'pocket_model_provider.dart'; +import 'pocket_voice_worker.dart'; +import 'voice_audio_output.dart'; + +enum PocketVoicePhase { off, loading, listening, synthesizing, speaking, error } + +enum PocketVoiceFailureKind { load, synthesis, playback } + +@immutable +class PocketVoiceState { + final PocketVoicePhase phase; + final String? conversationKey; + final PocketVoiceFailureKind? failureKind; + final String? error; + + const PocketVoiceState({ + this.phase = PocketVoicePhase.off, + this.conversationKey, + this.failureKind, + this.error, + }); + + bool get enabled => phase != PocketVoicePhase.off; +} + +final voiceAudioOutputProvider = Provider( + (_) => PlatformVoiceAudioOutput(), +); + +final pocketVoiceWorkerFactoryProvider = Provider( + (_) => PocketVoiceWorker.new, +); + +final pocketVoiceProvider = + NotifierProvider( + PocketVoiceNotifier.new, + ); + +class PocketVoiceNotifier extends Notifier { + final Queue _utterances = Queue(); + final Queue _audio = Queue(); + PocketVoiceWorker? _worker; + Future? _workerStart; + StreamSubscription? _workerSubscription; + StreamSubscription? _audioSubscription; + int _transitionEpoch = 0; + int _nextGeneration = 0; + int? _activeGeneration; + bool _synthesisComplete = false; + bool _playbackActive = false; + bool _stopping = false; + bool _queueWhileStopping = false; + Future? _stopFuture; + + @override + PocketVoiceState build() { + ref.listen(relayConfigProvider, (previous, _) { + if (previous != null) unawaited(disable()); + }); + _audioSubscription = ref + .read(voiceAudioOutputProvider) + .events + .listen(_handleAudioEvent); + ref.onDispose(() { + _transitionEpoch += 1; + _workerSubscription?.cancel(); + _audioSubscription?.cancel(); + _worker?.cancel(); + unawaited(_worker?.dispose()); + }); + return const PocketVoiceState(); + } + + Future enable(String conversationKey) async { + if (state.conversationKey == conversationKey && state.enabled) return; + final epoch = ++_transitionEpoch; + await _stopConversation(preserveIncoming: false); + if (epoch != _transitionEpoch) return; + + final model = ref.read(pocketModelProvider); + if (model.phase != PocketModelPhase.ready || model.path == null) { + throw StateError('Download Pocket voice before starting a conversation.'); + } + state = PocketVoiceState( + phase: PocketVoicePhase.loading, + conversationKey: conversationKey, + ); + try { + await _ensureWorker(model.path!); + if (epoch != _transitionEpoch) return; + state = PocketVoiceState( + phase: PocketVoicePhase.listening, + conversationKey: conversationKey, + ); + _startNextUtterance(); + } catch (error) { + if (epoch == _transitionEpoch) { + state = PocketVoiceState( + phase: PocketVoicePhase.error, + conversationKey: conversationKey, + failureKind: PocketVoiceFailureKind.load, + error: error.toString(), + ); + } + rethrow; + } + } + + Future disable() async { + _transitionEpoch += 1; + state = const PocketVoiceState(); + await _stopConversation(preserveIncoming: false); + } + + void speak(String conversationKey, String text) { + if (!state.enabled || state.conversationKey != conversationKey) return; + if (state.phase == PocketVoicePhase.error) return; + if (_stopping && !_queueWhileStopping) return; + final trimmed = text.trim(); + if (trimmed.length <= 1 || trimmed.startsWith('[System]')) return; + _utterances.add(trimmed); + _startNextUtterance(); + } + + Future interrupt() async { + final epoch = ++_transitionEpoch; + await _stopConversation(preserveIncoming: true); + if (epoch == _transitionEpoch && state.enabled) { + state = PocketVoiceState( + phase: PocketVoicePhase.listening, + conversationKey: state.conversationKey, + ); + _startNextUtterance(); + } + } + + Future _ensureWorker(String modelPath) { + final worker = _worker; + if (worker != null && worker.isReady) return Future.value(worker); + final starting = _workerStart; + if (starting != null) return starting; + + final created = ref.read(pocketVoiceWorkerFactoryProvider)(); + _worker = created; + _workerSubscription = created.responses.listen(_handleWorkerResponse); + final future = created.start(modelPath).then((_) => created); + _workerStart = future; + return () async { + try { + return await future; + } catch (error, stackTrace) { + if (identical(_worker, created)) { + _worker = null; + await _workerSubscription?.cancel(); + _workerSubscription = null; + } + await created.dispose(); + Error.throwWithStackTrace(error, stackTrace); + } finally { + if (identical(_workerStart, future)) _workerStart = null; + } + }(); + } + + Future _stopConversation({required bool preserveIncoming}) async { + final activeStop = _stopFuture; + if (activeStop != null) { + if (!preserveIncoming) { + _queueWhileStopping = false; + _utterances.clear(); + } + await activeStop; + return; + } + _stopping = true; + _queueWhileStopping = preserveIncoming; + _utterances.clear(); + _audio.clear(); + _activeGeneration = null; + _synthesisComplete = false; + _playbackActive = false; + final output = ref.read(voiceAudioOutputProvider); + final worker = _worker; + worker?.cancel(); + final stop = Future.wait([ + output.stop(), + if (worker != null) worker.cancelAndWait(), + ]); + _stopFuture = stop; + try { + await stop; + } finally { + if (identical(_stopFuture, stop)) { + _stopFuture = null; + _stopping = false; + _queueWhileStopping = false; + } + } + } + + void _startNextUtterance() { + if (_stopping || + _activeGeneration != null || + _utterances.isEmpty || + !state.enabled) { + return; + } + final worker = _worker; + if (worker == null || !worker.isReady) return; + final utterance = _utterances.removeFirst(); + final generation = ++_nextGeneration; + _activeGeneration = generation; + _synthesisComplete = false; + state = PocketVoiceState( + phase: PocketVoicePhase.synthesizing, + conversationKey: state.conversationKey, + ); + try { + worker.synthesize(generation, utterance); + } catch (error) { + _activeGeneration = null; + state = PocketVoiceState( + phase: PocketVoicePhase.error, + conversationKey: state.conversationKey, + failureKind: PocketVoiceFailureKind.synthesis, + error: error.toString(), + ); + } + } + + void _handleWorkerResponse(PocketWorkerResponse response) { + switch (response) { + case PocketWorkerReady(): + case PocketWorkerStopped(): + return; + case PocketWorkerDone(): + if (response.generation != _activeGeneration) return; + _synthesisComplete = true; + if (!_playbackActive && _audio.isEmpty) _finishUtterance(); + case PocketWorkerFailure(): + if (response.generation != _activeGeneration) return; + if (response.kind == PocketWorkerFailureKind.cancelled) { + _finishUtterance(); + return; + } + _failPlayback( + response.message, + failureKind: PocketVoiceFailureKind.synthesis, + ); + case PocketWorkerAudio(): + if (response.generation != _activeGeneration) return; + _audio.add(response); + if (response.isLast) _synthesisComplete = true; + unawaited(_playNextChunk()); + } + } + + Future _playNextChunk() async { + if (_playbackActive || _audio.isEmpty || _activeGeneration == null) return; + final activeGeneration = _activeGeneration; + final conversationKey = state.conversationKey; + final chunk = _audio.removeFirst(); + _playbackActive = true; + final output = ref.read(voiceAudioOutputProvider); + try { + await output.play( + chunk.data.materialize().asUint8List(), + chunk.sampleRate, + ); + } catch (error) { + if (activeGeneration == _activeGeneration && state.enabled) { + _failPlayback( + error.toString(), + failureKind: PocketVoiceFailureKind.playback, + ); + } + return; + } + if (activeGeneration != _activeGeneration || + !state.enabled || + state.conversationKey != conversationKey) { + _playbackActive = false; + await output.stop(); + return; + } + } + + void _handleAudioEvent(VoiceAudioEvent event) { + switch (event) { + case VoiceAudioEvent.started: + if (!_playbackActive || !state.enabled) return; + state = PocketVoiceState( + phase: PocketVoicePhase.speaking, + conversationKey: state.conversationKey, + ); + case VoiceAudioEvent.completed: + if (!_playbackActive) return; + _playbackActive = false; + if (_audio.isNotEmpty) { + unawaited(_playNextChunk()); + } else if (_synthesisComplete) { + _finishUtterance(); + } else if (state.enabled) { + state = PocketVoiceState( + phase: PocketVoicePhase.synthesizing, + conversationKey: state.conversationKey, + ); + } + case VoiceAudioEvent.error: + if (_playbackActive) { + _failPlayback( + 'Pocket voice playback failed.', + failureKind: PocketVoiceFailureKind.playback, + ); + } + case VoiceAudioEvent.interrupted: + case VoiceAudioEvent.routeLost: + case VoiceAudioEvent.backgrounded: + unawaited(interrupt()); + } + } + + void _failPlayback( + String message, { + required PocketVoiceFailureKind failureKind, + }) { + _worker?.cancel(); + _utterances.clear(); + _activeGeneration = null; + _audio.clear(); + _synthesisComplete = false; + _playbackActive = false; + unawaited(ref.read(voiceAudioOutputProvider).stop()); + if (state.enabled) { + state = PocketVoiceState( + phase: PocketVoicePhase.error, + conversationKey: state.conversationKey, + failureKind: failureKind, + error: message, + ); + } + } + + void _finishUtterance() { + _activeGeneration = null; + _synthesisComplete = false; + _playbackActive = false; + if (_utterances.isNotEmpty) { + _startNextUtterance(); + } else if (state.enabled) { + state = PocketVoiceState( + phase: PocketVoicePhase.listening, + conversationKey: state.conversationKey, + ); + } + } +} diff --git a/mobile/lib/shared/voice/pocket_voice_ffi.dart b/mobile/lib/shared/voice/pocket_voice_ffi.dart new file mode 100644 index 0000000000..82d68d4082 --- /dev/null +++ b/mobile/lib/shared/voice/pocket_voice_ffi.dart @@ -0,0 +1,166 @@ +import 'dart:convert'; +import 'dart:ffi' as ffi; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; + +final class _EngineResult extends ffi.Struct { + external ffi.Pointer engine; + external ffi.Pointer error; +} + +final class _PcmResult extends ffi.Struct { + external ffi.Pointer samples; + + @ffi.IntPtr() + external int len; + + @ffi.Uint32() + external int sampleRate; + + external ffi.Pointer error; +} + +typedef _CreateNative = _EngineResult Function(ffi.Pointer, ffi.Uint8); +typedef _CreateDart = _EngineResult Function(ffi.Pointer, int); +typedef _SynthNative = + _PcmResult Function(ffi.Pointer, ffi.Pointer); +typedef _SynthDart = + _PcmResult Function(ffi.Pointer, ffi.Pointer); +typedef _CancelNative = ffi.Void Function(ffi.Pointer); +typedef _CancelDart = void Function(ffi.Pointer); +typedef _ResetCancelNative = ffi.Void Function(ffi.Pointer); +typedef _ResetCancelDart = void Function(ffi.Pointer); +typedef _PrepareNative = + ffi.Pointer Function(ffi.Pointer, ffi.Pointer); +typedef _PrepareDart = + ffi.Pointer Function(ffi.Pointer, ffi.Pointer); +typedef _DestroyNative = ffi.Void Function(ffi.Pointer); +typedef _DestroyDart = void Function(ffi.Pointer); +typedef _FreePcmNative = ffi.Void Function(_PcmResult); +typedef _FreePcmDart = void Function(_PcmResult); +typedef _FreeStringNative = ffi.Void Function(ffi.Pointer); +typedef _FreeStringDart = void Function(ffi.Pointer); + +class PocketVoicePcm { + final Uint8List bytes; + final int sampleRate; + + const PocketVoicePcm(this.bytes, this.sampleRate); +} + +class PocketVoiceFfi { + static const fp32Precision = 0; + + final ffi.DynamicLibrary _library; + late final _CreateDart _create; + late final _SynthDart _synthesize; + late final _CancelDart _cancel; + late final _ResetCancelDart _resetCancel; + late final _PrepareDart _prepare; + late final _DestroyDart _destroy; + late final _FreePcmDart _freePcm; + late final _FreeStringDart _freeString; + + PocketVoiceFfi() : _library = _openLibrary() { + _create = _library.lookupFunction<_CreateNative, _CreateDart>( + 'buzz_voice_engine_create', + ); + _synthesize = _library.lookupFunction<_SynthNative, _SynthDart>( + 'buzz_voice_engine_synthesize', + ); + _cancel = _library.lookupFunction<_CancelNative, _CancelDart>( + 'buzz_voice_engine_cancel', + ); + _resetCancel = _library + .lookupFunction<_ResetCancelNative, _ResetCancelDart>( + 'buzz_voice_engine_reset_cancel', + ); + _prepare = _library.lookupFunction<_PrepareNative, _PrepareDart>( + 'buzz_voice_prepare_chunks_json', + ); + _destroy = _library.lookupFunction<_DestroyNative, _DestroyDart>( + 'buzz_voice_engine_destroy', + ); + _freePcm = _library.lookupFunction<_FreePcmNative, _FreePcmDart>( + 'buzz_voice_pcm_free', + ); + _freeString = _library.lookupFunction<_FreeStringNative, _FreeStringDart>( + 'buzz_voice_string_free', + ); + } + + int create(String modelPath) { + final path = modelPath.toNativeUtf8(); + final result = _create(path, fp32Precision); + calloc.free(path); + if (result.error != ffi.nullptr) { + throw StateError(_takeError(result.error)); + } + if (result.engine == ffi.nullptr) { + throw StateError('Pocket engine creation returned a null handle.'); + } + return result.engine.address; + } + + PocketVoicePcm synthesize(int handle, String text) { + final input = text.toNativeUtf8(); + final result = _synthesize( + ffi.Pointer.fromAddress(handle), + input, + ); + calloc.free(input); + try { + if (result.error != ffi.nullptr) { + throw StateError(result.error.cast().toDartString()); + } + if (result.samples == ffi.nullptr) { + throw StateError('Pocket synthesis returned a null sample buffer.'); + } + final sampleBytes = result.samples.cast().asTypedList( + result.len * 2, + ); + return PocketVoicePcm(Uint8List.fromList(sampleBytes), result.sampleRate); + } finally { + _freePcm(result); + } + } + + void cancel(int handle) => _cancel(ffi.Pointer.fromAddress(handle)); + + void resetCancel(int handle) => + _resetCancel(ffi.Pointer.fromAddress(handle)); + + List prepareChunks(int handle, String text) { + final input = text.toNativeUtf8(); + final result = _prepare(ffi.Pointer.fromAddress(handle), input); + calloc.free(input); + if (result == ffi.nullptr) { + throw StateError('Pocket text preparation returned no result.'); + } + final encoded = _takeError(result); + final decoded = jsonDecode(encoded); + if (decoded is! List) { + throw StateError('Pocket text preparation returned invalid JSON.'); + } + return decoded.cast(); + } + + void destroy(int handle) => + _destroy(ffi.Pointer.fromAddress(handle)); + + String _takeError(ffi.Pointer error) { + final message = error.cast().toDartString(); + _freeString(error); + return message; + } +} + +ffi.DynamicLibrary _openLibrary() { + if (Platform.isIOS) return ffi.DynamicLibrary.process(); + if (Platform.isAndroid) { + return ffi.DynamicLibrary.open('libbuzz_voice_mobile.so'); + } + throw UnsupportedError('Pocket voice is supported only on iOS and Android.'); +} diff --git a/mobile/lib/shared/voice/pocket_voice_worker.dart b/mobile/lib/shared/voice/pocket_voice_worker.dart new file mode 100644 index 0000000000..54503cfd4a --- /dev/null +++ b/mobile/lib/shared/voice/pocket_voice_worker.dart @@ -0,0 +1,239 @@ +import 'dart:async'; +import 'dart:isolate'; + +import 'pocket_voice_ffi.dart'; + +sealed class PocketWorkerResponse { + const PocketWorkerResponse(); +} + +class PocketWorkerReady extends PocketWorkerResponse { + final int handle; + + const PocketWorkerReady(this.handle); +} + +class PocketWorkerAudio extends PocketWorkerResponse { + final int generation; + final TransferableTypedData data; + final int sampleRate; + final Duration synthesisTime; + final bool isLast; + + const PocketWorkerAudio({ + required this.generation, + required this.data, + required this.sampleRate, + required this.synthesisTime, + required this.isLast, + }); +} + +class PocketWorkerDone extends PocketWorkerResponse { + final int generation; + + const PocketWorkerDone(this.generation); +} + +enum PocketWorkerFailureKind { load, synthesis, cancelled } + +class PocketWorkerFailure extends PocketWorkerResponse { + final int? generation; + final PocketWorkerFailureKind kind; + final String message; + + const PocketWorkerFailure(this.kind, this.message, {this.generation}); +} + +class PocketWorkerStopped extends PocketWorkerResponse { + const PocketWorkerStopped(); +} + +sealed class _PocketWorkerCommand { + const _PocketWorkerCommand(); +} + +class _Synthesize extends _PocketWorkerCommand { + final int generation; + final String text; + + const _Synthesize(this.generation, this.text); +} + +class _Dispose extends _PocketWorkerCommand { + const _Dispose(); +} + +class PocketVoiceWorker { + final StreamController _responses = + StreamController.broadcast(); + Isolate? _isolate; + SendPort? _commands; + int? _handle; + PocketVoiceFfi? _mainFfi; + ReceivePort? _receive; + Completer? _activeSynthesis; + int? _activeGeneration; + + Stream get responses => _responses.stream; + bool get isReady => _commands != null; + + Future start(String modelPath) async { + if (_isolate != null) return; + final receive = ReceivePort(); + _receive = receive; + final messages = receive.asBroadcastStream(); + _isolate = await Isolate.spawn(_workerMain, ( + receive.sendPort, + modelPath, + ), debugName: 'buzz-pocket-voice'); + final first = await messages.first; + if (first is! (SendPort, PocketWorkerResponse)) { + throw StateError('Pocket worker sent an invalid startup response.'); + } + _commands = first.$1; + if (first.$2 is PocketWorkerFailure) { + _isolate?.kill(); + _isolate = null; + receive.close(); + _receive = null; + _commands = null; + throw StateError((first.$2 as PocketWorkerFailure).message); + } + final ready = first.$2 as PocketWorkerReady; + _handle = ready.handle; + _mainFfi = PocketVoiceFfi(); + messages.listen((message) { + if (message is! PocketWorkerResponse) return; + if (_finishesActiveSynthesis(message)) { + _activeSynthesis?.complete(); + _activeSynthesis = null; + _activeGeneration = null; + } + _responses.add(message); + }); + } + + void synthesize(int generation, String text) { + final commands = _commands; + final handle = _handle; + final ffi = _mainFfi; + if (commands == null || handle == null || ffi == null) { + throw StateError('Pocket worker is not ready.'); + } + if (_activeSynthesis != null) { + throw StateError('Pocket worker already has an active synthesis.'); + } + // Clear cancellation before publishing the command. Any cancel that + // arrives after this point belongs to this generation and must stay set. + ffi.resetCancel(handle); + _activeGeneration = generation; + _activeSynthesis = Completer(); + commands.send(_Synthesize(generation, text)); + } + + void cancel() { + final handle = _handle; + if (handle != null) _mainFfi?.cancel(handle); + } + + Future cancelAndWait() async { + final active = _activeSynthesis; + if (active == null) return; + cancel(); + await active.future; + } + + Future dispose() async { + if (_responses.isClosed) return; + await cancelAndWait(); + final commands = _commands; + _commands = null; + _handle = null; + _mainFfi = null; + if (commands != null) { + final stopped = _responses.stream.firstWhere( + (response) => response is PocketWorkerStopped, + ); + commands.send(const _Dispose()); + await stopped; + } + _isolate?.kill(); + _isolate = null; + _receive?.close(); + _receive = null; + await _responses.close(); + } + + bool _finishesActiveSynthesis(PocketWorkerResponse response) { + final activeGeneration = _activeGeneration; + if (activeGeneration == null) return false; + return switch (response) { + PocketWorkerAudio(:final generation, :final isLast) => + generation == activeGeneration && isLast, + PocketWorkerDone(:final generation) => generation == activeGeneration, + PocketWorkerFailure(:final generation) => generation == activeGeneration, + _ => false, + }; + } +} + +void _workerMain((SendPort, String) startup) { + final output = startup.$1; + final commands = ReceivePort(); + final ffi = PocketVoiceFfi(); + late final int handle; + try { + handle = ffi.create(startup.$2); + } catch (error) { + output.send(( + commands.sendPort, + PocketWorkerFailure(PocketWorkerFailureKind.load, error.toString()), + )); + commands.close(); + return; + } + output.send((commands.sendPort, PocketWorkerReady(handle))); + commands.listen((command) { + switch (command) { + case _Synthesize(): + try { + final chunks = ffi.prepareChunks(handle, command.text); + if (chunks.isEmpty) { + output.send(PocketWorkerDone(command.generation)); + return; + } + for (var index = 0; index < chunks.length; index += 1) { + final stopwatch = Stopwatch()..start(); + final pcm = ffi.synthesize(handle, chunks[index]); + stopwatch.stop(); + output.send( + PocketWorkerAudio( + generation: command.generation, + data: TransferableTypedData.fromList([pcm.bytes]), + sampleRate: pcm.sampleRate, + synthesisTime: stopwatch.elapsed, + isLast: index == chunks.length - 1, + ), + ); + } + } catch (error) { + final message = error.toString(); + output.send( + PocketWorkerFailure( + message.contains('synthesis cancelled') + ? PocketWorkerFailureKind.cancelled + : PocketWorkerFailureKind.synthesis, + message, + generation: command.generation, + ), + ); + } + case _Dispose(): + ffi.destroy(handle); + commands.close(); + output.send(const PocketWorkerStopped()); + Isolate.exit(); + } + }); +} diff --git a/mobile/lib/shared/voice/voice_audio_output.dart b/mobile/lib/shared/voice/voice_audio_output.dart new file mode 100644 index 0000000000..2b65d680ba --- /dev/null +++ b/mobile/lib/shared/voice/voice_audio_output.dart @@ -0,0 +1,48 @@ +import 'dart:async'; + +import 'package:flutter/services.dart'; + +enum VoiceAudioEvent { + started, + completed, + interrupted, + routeLost, + backgrounded, + error, +} + +abstract interface class VoiceAudioOutput { + Stream get events; + Future play(Uint8List pcm, int sampleRate); + Future stop(); +} + +class PlatformVoiceAudioOutput implements VoiceAudioOutput { + static const _channel = MethodChannel('buzz/voice_audio'); + final StreamController _events = + StreamController.broadcast(); + + PlatformVoiceAudioOutput() { + _channel.setMethodCallHandler((call) async { + final event = switch (call.method) { + 'started' => VoiceAudioEvent.started, + 'completed' => VoiceAudioEvent.completed, + 'interrupted' => VoiceAudioEvent.interrupted, + 'routeLost' => VoiceAudioEvent.routeLost, + 'backgrounded' => VoiceAudioEvent.backgrounded, + _ => VoiceAudioEvent.error, + }; + _events.add(event); + }); + } + + @override + Stream get events => _events.stream; + + @override + Future play(Uint8List pcm, int sampleRate) => + _channel.invokeMethod('play', {'pcm': pcm, 'sampleRate': sampleRate}); + + @override + Future stop() => _channel.invokeMethod('stop'); +} diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 55185fc9c6..f8ea7973f2 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -346,7 +346,7 @@ packages: source: hosted version: "1.3.3" ffi: - dependency: transitive + dependency: "direct main" description: name: ffi sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" @@ -446,6 +446,11 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_driver: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" flutter_hooks: dependency: "direct main" description: @@ -568,6 +573,11 @@ packages: url: "https://pub.dev" source: hosted version: "4.0.0" + fuchsia_remote_debug_protocol: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" glob: dependency: transitive description: @@ -712,6 +722,11 @@ packages: url: "https://pub.dev" source: hosted version: "0.2.2" + integration_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" intl: dependency: "direct main" description: @@ -1032,6 +1047,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.5.2" + process: + dependency: transitive + description: + name: process + sha256: c6248e4526673988586e8c00bb22a49210c258dc91df5227d5da9748ecf79744 + url: "https://pub.dev" + source: hosted + version: "5.0.5" provider: dependency: transitive description: @@ -1261,6 +1284,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.1" + sync_http: + dependency: transitive + description: + name: sync_http + sha256: "7f0cd72eca000d2e026bcd6f990b81d0ca06022ef4e32fb257b30d3d1014a961" + url: "https://pub.dev" + source: hosted + version: "0.3.1" term_glyph: dependency: transitive description: @@ -1493,6 +1524,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.3" + webdriver: + dependency: transitive + description: + name: webdriver + sha256: "2f3a14ca026957870cfd9c635b83507e0e51d8091568e90129fbf805aba7cade" + url: "https://pub.dev" + source: hosted + version: "3.1.0" webkit_inspection_protocol: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index b90f50c25a..7d527a8b6a 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -9,6 +9,7 @@ environment: dependencies: flutter: sdk: flutter + ffi: ^2.1.4 hooks_riverpod: ^3.0.3 flutter_hooks: ^0.21.3 lucide_icons_flutter: ^3.1.0 @@ -40,6 +41,8 @@ dependencies: dev_dependencies: flutter_test: sdk: flutter + integration_test: + sdk: flutter flutter_lints: ^6.0.0 custom_lint: ^0.8.0 riverpod_lint: ^3.1.0 diff --git a/mobile/test/features/channels/channel_detail_page_test.dart b/mobile/test/features/channels/channel_detail_page_test.dart index af39eb103a..82f226e52a 100644 --- a/mobile/test/features/channels/channel_detail_page_test.dart +++ b/mobile/test/features/channels/channel_detail_page_test.dart @@ -482,6 +482,8 @@ void main() { expect(find.text('Forum threads are not on mobile yet'), findsNothing); // The compose bar for stream messages should not appear. expect(find.text('Message…'), findsNothing); + // Forum posts are not part of the channel-level voice conversation. + expect(find.byTooltip('Start Pocket voice'), findsNothing); }); testWidgets('renders video attachments from imeta tags in the timeline', ( diff --git a/mobile/test/features/channels/pocket_voice_conversation_test.dart b/mobile/test/features/channels/pocket_voice_conversation_test.dart new file mode 100644 index 0000000000..8c5fa8610f --- /dev/null +++ b/mobile/test/features/channels/pocket_voice_conversation_test.dart @@ -0,0 +1,147 @@ +import 'package:buzz/features/channels/channel_management_provider.dart'; +import 'package:buzz/features/channels/pocket_voice_conversation.dart'; +import 'package:buzz/shared/relay/relay.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + final bot = ChannelMember( + pubkey: 'bot', + role: 'bot', + joinedAt: DateTime.utc(2026), + ); + final person = ChannelMember( + pubkey: 'person', + role: 'member', + joinedAt: DateTime.utc(2026), + ); + + test('baselines history, then selects only new authoritative bot text', () { + final conversation = PocketVoiceConversation(); + expect( + conversation.update( + events: [_event('history', 'bot', 'old')], + members: [bot, person], + currentPubkey: 'self', + ), + isEmpty, + ); + + final spoken = conversation.update( + events: [ + _event('history', 'bot', 'old'), + _event('person', 'person', 'hello'), + _event('self', 'self', 'steer'), + _event('system', 'bot', '[System] working'), + _event('bot', 'bot', 'Assistant answer'), + ], + members: [bot, person], + currentPubkey: 'self', + ); + + expect(spoken.map((event) => event.id), ['bot']); + }); + + test('fails closed while membership is unresolved', () { + final conversation = PocketVoiceConversation(); + conversation.update(events: const [], members: null, currentPubkey: 'self'); + expect( + conversation.update( + events: [_event('bot', 'bot', 'answer')], + members: null, + currentPubkey: 'self', + ), + isEmpty, + ); + expect( + conversation.update( + events: [_event('bot', 'bot', 'answer')], + members: [bot], + currentPubkey: 'self', + ), + isEmpty, + ); + }); + + test('speaks new stream-message v2 bot replies', () { + final conversation = PocketVoiceConversation(); + conversation.update( + events: const [], + members: [bot], + currentPubkey: 'self', + ); + + final spoken = conversation.update( + events: [ + _event( + 'v2', + 'bot', + 'Assistant answer', + kind: EventKind.streamMessageV2, + ), + ], + members: [bot], + currentPubkey: 'self', + ); + + expect(spoken.map((event) => event.id), ['v2']); + }); + + test('thread conversation accepts only direct replies to its head', () { + final conversation = PocketVoiceConversation(); + conversation.update( + events: const [], + members: [bot], + currentPubkey: 'self', + threadHeadId: 'root', + ); + + final spoken = conversation.update( + events: [ + _event('top', 'bot', 'top level'), + _event('direct', 'bot', 'direct', parent: 'root'), + _event('nested', 'bot', 'nested', parent: 'other'), + ], + members: [bot], + currentPubkey: 'self', + threadHeadId: 'root', + ); + + expect(spoken.map((event) => event.id), ['direct']); + }); + + test('never re-speaks retained history after large conversation updates', () { + final conversation = PocketVoiceConversation(); + final history = [ + for (var index = 0; index < 1100; index += 1) + _event('history-$index', 'bot', 'Historical response $index'), + ]; + conversation.update(events: history, members: [bot], currentPubkey: 'self'); + + final spoken = conversation.update( + events: [...history, _event('new', 'bot', 'New response')], + members: [bot], + currentPubkey: 'self', + ); + + expect(spoken.map((event) => event.id), ['new']); + }); +} + +NostrEvent _event( + String id, + String pubkey, + String content, { + String? parent, + int kind = EventKind.streamMessage, +}) => NostrEvent( + id: id, + pubkey: pubkey, + createdAt: 1, + kind: kind, + tags: [ + const ['h', 'channel'], + if (parent != null) ['e', parent, '', 'reply'], + ], + content: content, + sig: '', +); diff --git a/mobile/test/features/channels/send_message_provider_test.dart b/mobile/test/features/channels/send_message_provider_test.dart index f91ce87b2b..951f004de7 100644 --- a/mobile/test/features/channels/send_message_provider_test.dart +++ b/mobile/test/features/channels/send_message_provider_test.dart @@ -66,6 +66,30 @@ void main() { expect(completedIds, isEmpty); expect(removedIds, [localMessages.single.id]); }); + + test('interrupts voice before publishing submitted user text', () async { + final session = _PendingPublishRelaySession(); + var interrupted = false; + final send = SendMessage( + signedEventRelay: SignedEventRelay( + session: session, + nsec: nostr.Keys.generate().nsec, + ), + fetchMembers: (_) async => const [], + readUserCache: () => const {}, + addLocalMessage: (_, _) { + expect(interrupted, isTrue); + }, + completeLocalMessage: (_, _) {}, + removeLocalMessage: (_, _) {}, + onBeforeSend: () async => interrupted = true, + ); + + final result = send(channelId: _channelId, content: 'steer'); + await session.published; + session.accept(); + await result; + }); } const _channelId = '11111111-1111-4111-8111-111111111111'; diff --git a/mobile/test/features/settings/goldens/pocket_voice_settings.png b/mobile/test/features/settings/goldens/pocket_voice_settings.png new file mode 100644 index 0000000000..54df8d67da Binary files /dev/null and b/mobile/test/features/settings/goldens/pocket_voice_settings.png differ diff --git a/mobile/test/features/settings/pocket_voice_settings_test.dart b/mobile/test/features/settings/pocket_voice_settings_test.dart new file mode 100644 index 0000000000..0f0bbe3b6a --- /dev/null +++ b/mobile/test/features/settings/pocket_voice_settings_test.dart @@ -0,0 +1,75 @@ +import 'dart:io'; + +import 'package:buzz/features/settings/settings_page.dart'; +import 'package:buzz/shared/theme/theme.dart'; +import 'package:buzz/shared/voice/pocket_model_provider.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:package_info_plus/package_info_plus.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +import '../../helpers/widget_helpers.dart'; + +void main() { + testWidgets('shows post-install Pocket model download details', ( + tester, + ) async { + SharedPreferences.setMockInitialValues({}); + PackageInfo.setMockInitialValues( + appName: 'Buzz', + packageName: 'xyz.block.buzz.mobile', + version: '1.0.0', + buildNumber: '1', + buildSignature: '', + ); + final preferences = await SharedPreferences.getInstance(); + await (FontLoader( + 'Inter', + )..addFont(rootBundle.load('assets/fonts/InterVariable.ttf'))).load(); + await (FontLoader('packages/lucide_icons_flutter/Lucide')..addFont( + rootBundle.load('packages/lucide_icons_flutter/assets/lucide.ttf'), + )) + .load(); + const screenshotKey = Key('pocket-voice-settings-screenshot'); + await tester.binding.setSurfaceSize(const Size(430, 932)); + addTearDown(() => tester.binding.setSurfaceSize(null)); + + await tester.pumpWidget( + WidgetHelpers.testable( + overrides: [ + savedPrefsProvider.overrideWithValue(preferences), + pocketModelProvider.overrideWith(_AbsentPocketModelNotifier.new), + ], + child: const RepaintBoundary( + key: screenshotKey, + child: SettingsPage(profileHeader: SizedBox.shrink()), + ), + ), + ); + await tester.pump(); + await tester.pump(const Duration(milliseconds: 500)); + + expect(find.text('Pocket voice'), findsOneWidget); + expect(find.text('Not downloaded'), findsOneWidget); + expect(find.textContaining('Downloads 440 MB'), findsOneWidget); + expect(find.text('440 MB'), findsOneWidget); + + // Flutter's text rasterization differs between macOS and the Linux CI + // runner even with the same bundled fonts. Keep the pixel baseline on the + // platform that owns the pixel baseline; the assertions above + // remain the cross-platform contract for the download state. + if (Platform.isMacOS) { + await expectLater( + find.byKey(screenshotKey), + matchesGoldenFile('goldens/pocket_voice_settings.png'), + ); + } + }); +} + +class _AbsentPocketModelNotifier extends PocketModelNotifier { + @override + PocketModelState build() => + const PocketModelState(phase: PocketModelPhase.absent); +} diff --git a/mobile/test/shared/voice/pocket_model_downloader_test.dart b/mobile/test/shared/voice/pocket_model_downloader_test.dart new file mode 100644 index 0000000000..a0a1e0479a --- /dev/null +++ b/mobile/test/shared/voice/pocket_model_downloader_test.dart @@ -0,0 +1,297 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:buzz/shared/voice/pocket_model_downloader.dart'; +import 'package:buzz/shared/voice/pocket_model_manifest.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; + +void main() { + late Directory temporary; + late PocketModelArtifact artifact; + + setUp(() async { + temporary = await Directory.systemTemp.createTemp('buzz-pocket-model-'); + artifact = PocketModelArtifact( + name: 'tiny.bin', + url: Uri.parse('https://example.test/tiny.bin'), + size: 5, + sha256: + '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e730' + '43362938b9824', + ); + }); + + tearDown(() async { + if (await temporary.exists()) await temporary.delete(recursive: true); + }); + + test( + 'installs verified files atomically and reports exact progress', + () async { + final excluded = []; + final progress = <(int, int, String)>[]; + final downloader = _downloader( + temporary, + artifact, + client: MockClient((_) async => http.Response('hello', 200)), + excludeFromBackup: excluded.add, + ); + final legacy = Directory('${temporary.path}/buzz/models/pocket-tts/v3'); + await legacy.create(recursive: true); + await File('${legacy.path}/sentinel').writeAsString('January'); + + final directory = await downloader.install( + (downloaded, total, file) => progress.add((downloaded, total, file)), + ); + + expect(await downloader.verify(directory, hashContents: true), isTrue); + expect(await File('${directory.path}/tiny.bin').readAsString(), 'hello'); + final manifest = + jsonDecode( + await File( + '${directory.path}/.buzz-model-manifest', + ).readAsString(), + ) + as Map; + expect(manifest['version'], pocketModelVersion); + expect(manifest['revision'], pocketModelRevision); + expect(manifest['precision'], pocketModelPrecision); + expect(progress.last, (5, 5, 'tiny.bin')); + expect(await legacy.exists(), isFalse); + expect(excluded, hasLength(2)); + expect( + excluded.first, + contains('.pocket-tts-v$pocketModelVersion.install-'), + ); + expect(excluded.last, directory.path); + expect( + await directory.parent + .list() + .where((entry) => entry.path.contains('.install-')) + .isEmpty, + isTrue, + ); + }, + ); + + test('rejects a checksum mismatch without replacing an install', () async { + final legacy = Directory('${temporary.path}/buzz/models/pocket-tts/v3'); + await legacy.create(recursive: true); + await File('${legacy.path}/sentinel').writeAsString('January'); + final downloader = _downloader( + temporary, + artifact, + client: MockClient((_) async => http.Response('wrong', 200)), + ); + + await expectLater( + downloader.install((_, _, _) {}), + throwsA(isA()), + ); + + expect( + await downloader.modelDirectory().then((dir) => dir.exists()), + isFalse, + ); + expect(await File('${legacy.path}/sentinel').readAsString(), 'January'); + }); + + test('fails before download when the model and reserve do not fit', () async { + final downloader = _downloader( + temporary, + artifact, + client: MockClient((_) async => http.Response('hello', 200)), + capacity: 4, + ); + + await expectLater( + downloader.install((_, _, _) {}), + throwsA( + isA() + .having((error) => error.available, 'available', 4) + .having((error) => error.required, 'required', greaterThan(5)), + ), + ); + }); + + test('cancellation closes the transfer and removes staging files', () async { + final client = _PendingClient(); + final downloader = _downloader(temporary, artifact, client: client); + + final install = downloader.install((_, _, _) {}); + await client.started.future; + downloader.cancel(); + + await expectLater(install, throwsA(isA())); + expect(client.closed, isTrue); + expect( + await downloader.modelDirectory().then((directory) => directory.exists()), + isFalse, + ); + }); + + test('recovers an old install left by an interrupted atomic swap', () async { + final parent = Directory('${temporary.path}/buzz/models/pocket-tts'); + final backup = Directory( + '${parent.path}/.pocket-tts-v$pocketModelVersion.old-crash', + ); + await backup.create(recursive: true); + await File('${backup.path}/sentinel').writeAsString('previous install'); + final client = _PendingClient(); + final downloader = _downloader(temporary, artifact, client: client); + + final install = downloader.install((_, _, _) {}); + await client.started.future; + final finalDirectory = await downloader.modelDirectory(); + expect( + await File('${finalDirectory.path}/sentinel').readAsString(), + 'previous install', + ); + downloader.cancel(); + await expectLater(install, throwsA(isA())); + }); + + test('replaces clients and retries when sending never completes', () async { + final clients = <_NeverSendingClient>[]; + final downloader = _downloader( + temporary, + artifact, + clientFactory: () { + final client = _NeverSendingClient(); + clients.add(client); + return client; + }, + sendTimeout: const Duration(milliseconds: 5), + ); + + await expectLater( + downloader.install((_, _, _) {}), + throwsA(isA()), + ); + + expect(clients, hasLength(3)); + expect(clients.every((client) => client.closed), isTrue); + }); + + test('replaces the client and retries after a body idle timeout', () async { + final stalled = _StallingBodyClient(); + final succeeding = MockClient((_) async => http.Response('hello', 200)); + final clients = [stalled, succeeding]; + var nextClient = 0; + final downloader = _downloader( + temporary, + artifact, + clientFactory: () => clients[nextClient++], + bodyIdleTimeout: const Duration(milliseconds: 5), + ); + + final directory = await downloader.install((_, _, _) {}); + + expect(await File('${directory.path}/tiny.bin').readAsString(), 'hello'); + expect(stalled.closed, isTrue); + expect(nextClient, 2); + }); + + test('April FP32 manifest matches the frozen shared runtime metadata', () { + expect(pocketModelVersion, '4'); + expect(pocketModelRevision, '58a6d00cf13d239b6748cb0769f35c580a8f606c'); + expect(pocketModelPrecision, 'fp32'); + expect( + pocketModelArtifacts + .where((artifact) => artifact.name != 'reference_sample.wav') + .where((artifact) => artifact.name != 'LICENSE') + .fold(0, (total, artifact) => total + artifact.size), + pocketModelCoreBytes, + ); + expect( + pocketModelArtifacts.fold(0, (total, artifact) => total + artifact.size), + pocketModelRuntimeBytes, + ); + expect( + pocketModelArtifacts.map((artifact) => artifact.name), + containsAll([ + 'bundle.json', + 'bos_before_voice.npy', + 'tokenizer.model', + 'flow_lm_main.onnx', + 'flow_lm_flow.onnx', + 'mimi_decoder.onnx', + 'mimi_encoder.onnx', + 'text_conditioner.onnx', + ]), + ); + expect(pocketModelLicenseText, contains('April 2026 ONNX export')); + expect(pocketModelLicenseText, contains(pocketModelRevision)); + }); +} + +PocketModelDownloader _downloader( + Directory support, + PocketModelArtifact artifact, { + http.Client? client, + http.Client Function()? clientFactory, + int capacity = 1024 * 1024 * 1024, + void Function(String)? excludeFromBackup, + Duration sendTimeout = const Duration(seconds: 30), + Duration bodyIdleTimeout = const Duration(seconds: 30), +}) => PocketModelDownloader( + clientFactory: clientFactory ?? () => client!, + applicationSupportDirectory: () async => support, + availableCapacity: (_) async => capacity, + excludeFromBackup: (path) async => excludeFromBackup?.call(path), + artifacts: [artifact], + installationId: 'test', + sendTimeout: sendTimeout, + bodyIdleTimeout: bodyIdleTimeout, +); + +class _PendingClient extends http.BaseClient { + final started = Completer(); + final _response = StreamController>(); + bool closed = false; + + @override + Future send(http.BaseRequest request) async { + started.complete(); + return http.StreamedResponse(_response.stream, 200, contentLength: 5); + } + + @override + void close() { + closed = true; + unawaited(_response.close()); + } +} + +class _NeverSendingClient extends http.BaseClient { + bool closed = false; + + @override + Future send(http.BaseRequest request) => + Completer().future; + + @override + void close() { + closed = true; + } +} + +class _StallingBodyClient extends http.BaseClient { + final _response = StreamController>(); + bool closed = false; + + @override + Future send(http.BaseRequest request) async { + scheduleMicrotask(() => _response.add('he'.codeUnits)); + return http.StreamedResponse(_response.stream, 200, contentLength: 5); + } + + @override + void close() { + closed = true; + unawaited(_response.close()); + } +} diff --git a/mobile/test/shared/voice/pocket_voice_controller_test.dart b/mobile/test/shared/voice/pocket_voice_controller_test.dart new file mode 100644 index 0000000000..e94ef83571 --- /dev/null +++ b/mobile/test/shared/voice/pocket_voice_controller_test.dart @@ -0,0 +1,324 @@ +import 'dart:async'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:buzz/shared/relay/relay.dart'; +import 'package:buzz/shared/voice/pocket_model_provider.dart'; +import 'package:buzz/shared/voice/pocket_voice_controller.dart'; +import 'package:buzz/shared/voice/pocket_voice_worker.dart'; +import 'package:buzz/shared/voice/voice_audio_output.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +void main() { + test('queues assistant messages and plays every chunk in order', () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + await notifier.enable('conversation'); + notifier.speak('conversation', 'First response.'); + notifier.speak('conversation', 'Second response.'); + + expect(worker.syntheses, [(1, 'First response.')]); + + worker.emitAudio(1, [1, 2], isLast: false); + await _flush(); + expect(output.played, hasLength(1)); + expect(output.played[0].$1, [1, 2]); + expect(output.played[0].$2, 24000); + expect( + container.read(pocketVoiceProvider).phase, + PocketVoicePhase.synthesizing, + ); + output.started(); + await _flush(); + expect( + container.read(pocketVoiceProvider).phase, + PocketVoicePhase.speaking, + ); + + worker.emitAudio(1, [3, 4], isLast: true); + await _flush(); + expect(output.played, hasLength(1)); + output.complete(); + await _flush(); + expect(output.played, hasLength(2)); + expect(output.played[1].$1, [3, 4]); + expect(worker.syntheses, [(1, 'First response.')]); + + output.complete(); + await _flush(); + expect(worker.syntheses, [(1, 'First response.'), (2, 'Second response.')]); + + worker.emitAudio(2, [5, 6], isLast: true); + await _flush(); + output.complete(); + await _flush(); + + expect(output.played, hasLength(3)); + expect(output.played[0].$1, [1, 2]); + expect(output.played[1].$1, [3, 4]); + expect(output.played[2].$1, [5, 6]); + expect( + container.read(pocketVoiceProvider).phase, + PocketVoicePhase.listening, + ); + }); + + test( + 'disable wins an overlapping engine startup and retains warm worker', + () async { + final worker = _FakeWorker(startPaused: true); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + final enabling = notifier.enable('conversation'); + await _flush(); + final disabling = notifier.disable(); + worker.finishStart(); + await Future.wait([enabling, disabling]); + + expect(container.read(pocketVoiceProvider).phase, PocketVoicePhase.off); + expect(worker.startCount, 1); + expect(worker.disposeCount, 0); + + await notifier.enable('next-conversation'); + expect(worker.startCount, 1); + expect( + container.read(pocketVoiceProvider).phase, + PocketVoicePhase.listening, + ); + }, + ); + + test( + 'preserves text submitted while the resident engine is loading', + () async { + final worker = _FakeWorker(startPaused: true); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + final longResponse = 'A' * 2100; + + final enabling = notifier.enable('conversation'); + await _flush(); + notifier.speak('conversation', longResponse); + worker.finishStart(); + await enabling; + + expect(worker.syntheses, [(1, longResponse)]); + }, + ); + + test( + 'queues new responses during interrupt and rejects them during disable', + () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + await notifier.enable('conversation'); + notifier.speak('conversation', 'Before interrupt.'); + worker.pauseCancellation(); + final interrupting = notifier.interrupt(); + await _flush(); + notifier.speak('conversation', 'After steering.'); + expect(worker.syntheses, [(1, 'Before interrupt.')]); + + worker.finishCancellation(); + await interrupting; + expect(worker.syntheses, [ + (1, 'Before interrupt.'), + (2, 'After steering.'), + ]); + + worker.pauseCancellation(); + final disabling = notifier.disable(); + notifier.speak('conversation', 'Must not be spoken.'); + worker.finishCancellation(); + await disabling; + expect(worker.syntheses, hasLength(2)); + expect(container.read(pocketVoiceProvider).phase, PocketVoicePhase.off); + }, + ); + + test('surfaces asynchronous native playback errors', () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + await notifier.enable('conversation'); + notifier.speak('conversation', 'Response.'); + worker.emitAudio(1, [1, 2], isLast: true); + await _flush(); + output.fail(); + await _flush(); + + final state = container.read(pocketVoiceProvider); + expect(state.phase, PocketVoicePhase.error); + expect(state.failureKind, PocketVoiceFailureKind.playback); + expect(state.error, 'Pocket voice playback failed.'); + expect(worker.cancelCount, greaterThan(0)); + + notifier.speak('conversation', 'Must wait for explicit recovery.'); + await _flush(); + expect(worker.syntheses, [(1, 'Response.')]); + expect(container.read(pocketVoiceProvider), same(state)); + }); + + test('classifies synthesis failures for platform fallback routing', () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + await notifier.enable('conversation'); + notifier.speak('conversation', 'Response.'); + worker.emitFailure( + 1, + PocketWorkerFailureKind.synthesis, + 'Pocket synthesis failed.', + ); + await _flush(); + + final state = container.read(pocketVoiceProvider); + expect(state.phase, PocketVoicePhase.error); + expect(state.failureKind, PocketVoiceFailureKind.synthesis); + expect(state.error, 'Pocket synthesis failed.'); + }); +} + +ProviderContainer _container(_FakeWorker worker, _FakeAudioOutput output) => + ProviderContainer( + overrides: [ + relayConfigProvider.overrideWith(_TestRelayConfigNotifier.new), + pocketModelProvider.overrideWith(_ReadyPocketModelNotifier.new), + pocketVoiceWorkerFactoryProvider.overrideWithValue(() => worker), + voiceAudioOutputProvider.overrideWithValue(output), + ], + ); + +Future _flush() => Future.delayed(Duration.zero); + +class _ReadyPocketModelNotifier extends PocketModelNotifier { + @override + PocketModelState build() => const PocketModelState( + phase: PocketModelPhase.ready, + path: '/tmp/pocket-model', + ); +} + +class _TestRelayConfigNotifier extends RelayConfigNotifier { + @override + RelayConfig build() => const RelayConfig(baseUrl: 'http://localhost:3000'); +} + +class _FakeWorker extends PocketVoiceWorker { + final StreamController _controller = + StreamController.broadcast(); + final Completer? _startGate; + final List<(int, String)> syntheses = []; + Completer? _cancelGate; + bool _ready = false; + int startCount = 0; + int disposeCount = 0; + int cancelCount = 0; + + _FakeWorker({bool startPaused = false}) + : _startGate = startPaused ? Completer() : null; + + @override + Stream get responses => _controller.stream; + + @override + bool get isReady => _ready; + + @override + Future start(String modelPath) async { + startCount += 1; + await _startGate?.future; + _ready = true; + } + + void finishStart() => _startGate?.complete(); + + @override + void synthesize(int generation, String text) { + syntheses.add((generation, text)); + } + + void emitAudio(int generation, List bytes, {required bool isLast}) { + _controller.add( + PocketWorkerAudio( + generation: generation, + data: TransferableTypedData.fromList([Uint8List.fromList(bytes)]), + sampleRate: 24000, + synthesisTime: const Duration(milliseconds: 1), + isLast: isLast, + ), + ); + } + + void emitFailure( + int generation, + PocketWorkerFailureKind kind, + String message, + ) { + _controller.add(PocketWorkerFailure(kind, message, generation: generation)); + } + + @override + void cancel() { + cancelCount += 1; + } + + @override + Future cancelAndWait() async { + await _cancelGate?.future; + _cancelGate = null; + } + + void pauseCancellation() => _cancelGate = Completer(); + + void finishCancellation() => _cancelGate?.complete(); + + @override + Future dispose() async { + disposeCount += 1; + await _controller.close(); + } +} + +class _FakeAudioOutput implements VoiceAudioOutput { + final StreamController _controller = + StreamController.broadcast(); + final List<(List, int)> played = []; + + @override + Stream get events => _controller.stream; + + @override + Future play(Uint8List pcm, int sampleRate) async { + played.add((pcm.toList(), sampleRate)); + } + + void complete() => _controller.add(VoiceAudioEvent.completed); + + void started() => _controller.add(VoiceAudioEvent.started); + + void fail() => _controller.add(VoiceAudioEvent.error); + + @override + Future stop() async {} +} diff --git a/scripts/mobile-voice-native.sh b/scripts/mobile-voice-native.sh new file mode 100755 index 0000000000..17ba2cb417 --- /dev/null +++ b/scripts/mobile-voice-native.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +version="1.13.4" +release_url="https://github.com/k2-fsa/sherpa-onnx/releases/download/v${version}" +cache_root="${BUZZ_VOICE_NATIVE_CACHE:-${HOME}/Library/Caches/buzz-mobile-voice/sherpa-onnx-v${version}}" +generated_root="${repo_root}/mobile/.generated/voice" + +android_archive="sherpa-onnx-v${version}-android.tar.bz2" +android_sha256="7983fc3de23f6e64148f2fb05fa94a2efaa8c0516cc1573383dc5c7d4d2a43b0" +ios_archive="sherpa-onnx-v${version}-ios.tar.bz2" +ios_sha256="596f33bff80046a52144745745fe54d55e8b23659d92209f5ab7d94c1259fe6d" + +usage() { + echo "usage: $0 fetch|build-ios|build-ios-current|build-android|build-all" >&2 + exit 2 +} + +verify_sha256() { + local path="$1" + local expected="$2" + local actual + actual="$(shasum -a 256 "$path" | awk '{print $1}')" + if [[ "$actual" != "$expected" ]]; then + echo "checksum mismatch for $path: expected $expected, got $actual" >&2 + return 1 + fi +} + +fetch_archive() { + local archive="$1" + local expected="$2" + mkdir -p "$cache_root" + if [[ -f "${cache_root}/${archive}" ]] && + ! verify_sha256 "${cache_root}/${archive}" "$expected"; then + rm "${cache_root:?}/${archive}" + fi + if [[ ! -f "${cache_root}/${archive}" ]]; then + curl --fail --location --retry 3 \ + --output "${cache_root}/${archive}.part" \ + "${release_url}/${archive}" + verify_sha256 "${cache_root}/${archive}.part" "$expected" + mv "${cache_root}/${archive}.part" "${cache_root}/${archive}" + fi +} + +extract_archive() { + local archive="$1" + local expected="$2" + local destination="$3" + fetch_archive "$archive" "$expected" + if [[ -f "${destination}/.complete" ]]; then + return + fi + local staging="${destination}.install-$$" + rm -rf "$staging" + mkdir -p "$staging" + tar xjf "${cache_root}/${archive}" -C "$staging" + touch "${staging}/.complete" + rm -rf "$destination" + mv "$staging" "$destination" +} + +fetch_native() { + extract_archive "$android_archive" "$android_sha256" "${cache_root}/android" + extract_archive "$ios_archive" "$ios_sha256" "${cache_root}/ios" +} + +ios_lib_dir() { + local sdk="$1" + local arch="$2" + local sherpa_slice + local ort_slice + if [[ "$sdk" == "iphoneos" ]]; then + sherpa_slice="ios-arm64" + ort_slice="ios-arm64" + else + sherpa_slice="ios-arm64_x86_64-simulator" + ort_slice="ios-arm64_x86_64-simulator" + fi + local destination="${generated_root}/ios/native/${sdk}-${arch}" + mkdir -p "$destination" + local sherpa_binary="${cache_root}/ios/build-ios/sherpa-onnx.xcframework/${sherpa_slice}/libsherpa-onnx.a" + if [[ "$sdk" == "iphoneos" ]]; then + cp "$sherpa_binary" "${destination}/libsherpa-onnx.a" + else + lipo "$sherpa_binary" -thin "$arch" -output "${destination}/libsherpa-onnx.a" + fi + local ort_binary="${cache_root}/ios/build-ios/ios-onnxruntime/1.27.0/onnxruntime.xcframework/${ort_slice}/onnxruntime.framework/onnxruntime" + lipo "$ort_binary" -thin "$arch" -output "${destination}/libonnxruntime.a" + printf '%s\n' "$destination" +} + +build_ios_target() { + local target="$1" + local sdk="$2" + local arch="$3" + local lib_dir + lib_dir="$(ios_lib_dir "$sdk" "$arch")" + IPHONEOS_DEPLOYMENT_TARGET=16.0 SHERPA_ONNX_LIB_DIR="$lib_dir" \ + cargo build --release -p buzz-voice-mobile --target "$target" + local destination="${generated_root}/ios/${sdk}-${arch}" + mkdir -p "$destination" + rm -f "$destination/libbuzz_voice_mobile.a" + cp "${repo_root}/target/${target}/release/libbuzz_voice_mobile.a" "$destination/" +} + +build_ios() { + fetch_native + build_ios_target "aarch64-apple-ios" "iphoneos" "arm64" + build_ios_target "aarch64-apple-ios-sim" "iphonesimulator" "arm64" + build_ios_target "x86_64-apple-ios" "iphonesimulator" "x86_64" +} + +build_ios_current() { + fetch_native + local platform="${PLATFORM_NAME:-${SDK_NAME%%[0-9]*}}" + local arch="${CURRENT_ARCH:-}" + if [[ -z "$arch" || "$arch" == "undefined_arch" ]]; then + if [[ " ${ARCHS:-} " == *" arm64 "* ]]; then + arch="arm64" + elif [[ " ${ARCHS:-} " == *" x86_64 "* ]]; then + arch="x86_64" + fi + fi + case "${platform}:${arch}" in + iphoneos:arm64) + build_ios_target "aarch64-apple-ios" "iphoneos" "arm64" + ;; + iphonesimulator:arm64) + build_ios_target "aarch64-apple-ios-sim" "iphonesimulator" "arm64" + ;; + iphonesimulator:x86_64) + build_ios_target "x86_64-apple-ios" "iphonesimulator" "x86_64" + ;; + *) + echo "unsupported iOS platform/architecture: ${platform:-unset}/${arch:-unset}" >&2 + exit 1 + ;; + esac +} + +android_target() { + case "$1" in + arm64-v8a) echo "aarch64-linux-android" ;; + x86_64) echo "x86_64-linux-android" ;; + *) return 1 ;; + esac +} + +build_android_abi() { + local abi="$1" + local target + target="$(android_target "$abi")" + local ndk_root="${ANDROID_NDK_HOME:-${ANDROID_NDK_ROOT:-}}" + if [[ -z "$ndk_root" ]]; then + echo "ANDROID_NDK_HOME must point at an Android NDK" >&2 + exit 1 + fi + local host_os + case "$(uname -s)" in + Darwin) host_os="darwin" ;; + Linux) host_os="linux" ;; + MINGW* | MSYS* | CYGWIN*) host_os="windows" ;; + *) + echo "Unsupported Android NDK host OS: $(uname -s)" >&2 + exit 1 + ;; + esac + local host_arch + case "$(uname -m)" in + arm64 | aarch64) host_arch="arm64" ;; + x86_64 | amd64) host_arch="x86_64" ;; + *) + echo "Unsupported Android NDK host architecture: $(uname -m)" >&2 + exit 1 + ;; + esac + local api="${BUZZ_ANDROID_MIN_API:-24}" + local clang_target + if [[ "$abi" == "arm64-v8a" ]]; then + clang_target="aarch64-linux-android${api}" + else + clang_target="x86_64-linux-android${api}" + fi + local prebuilt_root="${ndk_root}/toolchains/llvm/prebuilt" + local toolchain="${prebuilt_root}/${host_os}-${host_arch}" + local linker="${toolchain}/bin/${clang_target}-clang" + if [[ ! -x "$linker" ]]; then + local candidate + for candidate in "${prebuilt_root}/${host_os}-"*; do + if [[ -x "${candidate}/bin/${clang_target}-clang" ]]; then + toolchain="$candidate" + linker="${candidate}/bin/${clang_target}-clang" + break + fi + done + fi + if [[ ! -x "$linker" ]]; then + echo "Android linker not found: $linker" >&2 + exit 1 + fi + local source_libs="${cache_root}/android/jniLibs/${abi}" + local target_env + target_env="$(printf '%s' "$target" | tr '[:lower:]-' '[:upper:]_')" + local linker_var="CARGO_TARGET_${target_env}_LINKER" + local rustflags_var="CARGO_TARGET_${target_env}_RUSTFLAGS" + if ! rustup target list --installed | grep -Fxq "$target"; then + rustup target add "$target" + fi + env \ + SHERPA_ONNX_LIB_DIR="$source_libs" \ + "$linker_var=$linker" \ + "$rustflags_var=-L native=${source_libs}" \ + cargo rustc --release -p buzz-voice-mobile --no-default-features \ + --features shared --target "$target" --lib -- --crate-type cdylib + + local destination="${generated_root}/android/jniLibs/${abi}" + mkdir -p "$destination" + local shared_library="${repo_root}/target/${target}/release/libbuzz_voice_mobile.so" + if [[ ! -f "$shared_library" ]]; then + echo "Android cdylib was not produced for $target" >&2 + exit 1 + fi + cp "$shared_library" "${destination}/libbuzz_voice_mobile.so" + cp "${source_libs}/libsherpa-onnx-c-api.so" "$destination/" + cp "${source_libs}/libonnxruntime.so" "$destination/" +} + +build_android() { + fetch_native + build_android_abi "arm64-v8a" + build_android_abi "x86_64" +} + +case "${1:-}" in + fetch) fetch_native ;; + build-ios) build_ios ;; + build-ios-current) build_ios_current ;; + build-android) build_android ;; + build-all) + build_ios + build_android + ;; + *) usage ;; +esac