From 9b6fe53df145be69119099fc92aab82c0dc20a8f Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 27 Jul 2026 18:55:01 -0400 Subject: [PATCH 1/8] refactor(desktop): extract reusable Pocket voice primitives Signed-off-by: John Tennant --- Cargo.lock | 66 ++ Cargo.toml | 1 + Justfile | 1 + crates/buzz-voice/Cargo.toml | 12 + crates/buzz-voice/src/lib.rs | 3 + crates/buzz-voice/src/pocket.rs | 654 +++++++++++++++++ desktop/src-tauri/Cargo.lock | 9 + desktop/src-tauri/Cargo.toml | 1 + .../src-tauri/examples/pocket_quality_ab.rs | 2 +- desktop/src-tauri/src/huddle/pocket.rs | 655 +----------------- scripts/run-tests.sh | 3 + 11 files changed, 753 insertions(+), 654 deletions(-) create mode 100644 crates/buzz-voice/Cargo.toml create mode 100644 crates/buzz-voice/src/lib.rs create mode 100644 crates/buzz-voice/src/pocket.rs diff --git a/Cargo.lock b/Cargo.lock index 3b60dc4579..5a57e203d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1262,6 +1262,14 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "serde_json", + "sherpa-onnx", +] + [[package]] name = "buzz-workflow" version = "0.1.0" @@ -1329,6 +1337,26 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "castaway" version = "0.2.4" @@ -8049,6 +8077,28 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "sherpa-onnx" +version = "1.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b142d3f255cb4e4b7808ea25869db6f5714e0a3550da355234483b4db552055" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc951af03dc0653c0622158ca8a585a6f2bc43b7b06048cf0e5b5020005c227" +dependencies = [ + "bzip2", + "tar", + "ureq", +] + [[package]] name = "shlex" version = "1.3.0" @@ -9539,6 +9589,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.8" diff --git a/Cargo.toml b/Cargo.toml index 3ac7ee4cce..3268cfaf8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "crates/buzz-pair-relay", "crates/buzz-relay-mesh", "crates/buzz-dev-mcp", + "crates/buzz-voice", "examples/countdown-bot", ] exclude = ["desktop/src-tauri"] diff --git a/Justfile b/Justfile index bcef8983bc..e5601607ba 100644 --- a/Justfile +++ b/Justfile @@ -276,6 +276,7 @@ test-unit: #!/usr/bin/env bash if command -v cargo-nextest &>/dev/null; then cargo nextest run -p buzz-core -p buzz-auth --lib + cargo nextest run -p buzz-voice --lib # buzz-db migrator/lint tests: pure SQL-parsing unit tests (no infra). # They guard the embedded-migrator invariant (exactly the consolidated # 0001; cutover/backfill stays an operator script, not startup state) diff --git a/crates/buzz-voice/Cargo.toml b/crates/buzz-voice/Cargo.toml new file mode 100644 index 0000000000..f577e3d1a6 --- /dev/null +++ b/crates/buzz-voice/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "buzz-voice" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Reusable local voice primitives for Buzz" + +[dependencies] +serde_json = { workspace = true } +sherpa-onnx = "1.12" diff --git a/crates/buzz-voice/src/lib.rs b/crates/buzz-voice/src/lib.rs new file mode 100644 index 0000000000..e2ce3f255b --- /dev/null +++ b/crates/buzz-voice/src/lib.rs @@ -0,0 +1,3 @@ +//! Reusable local voice primitives for Buzz. + +pub mod pocket; diff --git a/crates/buzz-voice/src/pocket.rs b/crates/buzz-voice/src/pocket.rs new file mode 100644 index 0000000000..ee1faf928a --- /dev/null +++ b/crates/buzz-voice/src/pocket.rs @@ -0,0 +1,654 @@ +//! Pocket TTS engine wrapper around sherpa-onnx's `OfflineTts`. +//! +//! Pocket TTS is a small (~473 MB fp32 ONNX) zero-shot voice-cloning TTS +//! model from Kyutai. It runs quickly on CPU via sherpa-onnx, replacing the +//! previous Kokoro-82M engine that also required an espeak-free but +//! lexicon-heavy G2P pipeline (Misaki + CMUdict). +//! +//! Full-precision fp32 sessions, not the ~189 MB int8 quantization we +//! originally shipped: a direct same-runtime A/B (k2-fsa/sherpa-onnx#3172) +//! found the int8 ONNX export audibly degraded output quality, and fp32 +//! "significantly improved quality even at 1 step". +//! +//! ## Attribution +//! +//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. +//! arXiv:2509.06926. Original repository: . +//! Licensed CC-BY-4.0. +//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. +//! - **ONNX export**: KevinAHM — +//! . CC-BY-4.0. +//! - **sherpa-onnx repackage**: csukuangfj / k2-fsa — +//! . +//! Repackages KevinAHM's export with the file layout sherpa-onnx's +//! `OfflineTtsPocketModelConfig` expects. CC-BY-4.0. +//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary +//! (f, conversation)" preset from the Kyutai TTS demo +//! (), which maps to `vctk/p333_023_enhanced.wav` +//! in . CC-BY-4.0, base recording +//! from the VCTK corpus, enhanced by ai-coustics. +//! +//! Buzz ships these files unmodified; see the on-disk `MODEL_LICENSE.txt` +//! sidecar written by `huddle::models` during install for the canonical +//! CC-BY-4.0 §3(a)(1) attribution block. +//! +//! ## Engine-module contract (see `huddle::tts`) +//! +//! `pocket.rs` exposes a fixed surface used by `tts.rs`. Mirroring this +//! contract is what lets the TTS pipeline stay engine-agnostic: +//! +//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. +//! - `DEFAULT_VOICE: &str` — default voice name (without extension). +//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. +//! - `load_text_to_speech(model_dir)` → `Result` +//! - `load_voice_style(path)` → `Result` +//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` +//! → `Result, String>` +//! +//! `lang` and `steps` are accepted for API compatibility with the previous +//! Kokoro engine but are unused — Pocket TTS does its own language ID from +//! the input text and is not a diffusion model (consistency LM, one step). +//! There is no speed knob: sherpa-onnx's `GenerationConfig.speed` is only +//! read by some model families (vits), never by the Pocket impl +//! (`offline-tts-pocket-impl.h` — zero references), and upstream pocket-tts +//! has no speed parameter either. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; + +// ── Engine-module contract: public consts ───────────────────────────────────── + +/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, +/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. +pub const SAMPLE_RATE: u32 = 24_000; + +/// Name (without extension) of the bundled reference voice. The model directory +/// is expected to contain `.` after install. +pub const DEFAULT_VOICE: &str = "reference_sample"; + +/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the +/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, +/// not a precomputed embedding, so the extension change is honest. +pub const VOICE_FILE_EXT: &str = "wav"; + +// ── Tuning ──────────────────────────────────────────────────────────────────── + +/// Single-threaded ONNX execution for predictable CPU contention with the STT +/// pipeline. Matches `STT_NUM_THREADS` in `stt.rs`; raise only if a benchmark +/// argues for it. +const TTS_NUM_THREADS: i32 = 1; + +/// LRU cache size for cloned voice embeddings inside the sherpa-onnx engine. +/// We bind to one voice per pipeline today, but the upstream example uses 16 +/// and the cost is negligible — keep room for future multi-voice support. +const VOICE_EMBEDDING_CACHE_CAPACITY: i32 = 16; + +/// Pocket TTS is a consistency-based LM. Generation quality saturates at one +/// denoising step — the upstream `GenerationConfig` default of 5 multiplies +/// synthesis time by ~5× with no audible benefit on this model. +const SYNTH_NUM_STEPS: i32 = 1; + +/// Leave the generated audio's silences untouched (1.0 is the identity). +/// +/// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding +/// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and +/// multiplies its length by this factor. The previous value of 0.0 — set +/// under the mistaken belief it disabled lead-in/lead-out padding — deleted +/// every natural pause inside an utterance: clause breaks, breaths, the gap +/// after a comma. Words slammed together and endings cut abruptly. The +/// reference Pocket TTS pipeline does not post-process silence at all; +/// 1.0 restores parity. +const SYNTH_SILENCE_SCALE: f32 = 1.0; + +/// sherpa-onnx upstream default for `max_frames` (LM steps), in +/// `offline-tts-pocket-impl.h:Generate`. 500 steps ≈ 40 s of audio at the +/// Mimi 12.5 Hz frame rate. Referenced only by the regression test below; +/// production code path never raises (or even reads) this value — we just +/// leave sherpa-onnx's own default in place by not setting the override. +#[cfg(test)] +const SHERPA_ONNX_MAX_FRAMES_DEFAULT: i32 = 500; + +/// Tight `max_frames` we ask for on short, padded prompts to bound the +/// original "monster breathing" runaway. 100 LM steps ≈ 8 s of audio — +/// roomy for any one-to-four-word utterance the user is likely to elicit +/// while still well short of the 40 s upstream default. Chosen with slack so +/// we never *truncate* a legitimate short reply. +const SHORT_PROMPT_MAX_FRAMES: i32 = 100; + +/// Word-count threshold (inclusive) below which we pad the prompt with +/// leading spaces and cap `max_frames` tighter than the upstream default. +/// Matches upstream `pocket_tts.models.tts_model.prepare_text_prompt`. Above +/// this threshold we leave sherpa-onnx's own defaults in place — overriding +/// them caused the "first 'yep' is just static" regression seen on +/// 2026-05-18, where dropping `frames_after_eos` below the upstream default +/// of 3 clipped the leading audio of multi-clause sentences. +const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; + +/// Number of leading spaces prepended to short prompts. The upstream Python +/// uses exactly 8 — keep parity rather than tuning blindly. +/// +/// This is upstream's *only* mitigation for the FlowLM cold-start smear on +/// short utterances (kyutai-labs/pocket-tts #91, #70): the autoregressive +/// generation has a 2–3 step "settle" period where the first phoneme can be +/// smeared. A previous revision added a sacrificial `". . "` prefix plus an +/// amplitude-threshold trim to strip the rendered prefix from the output — +/// but the trim's absolute threshold (0.02 against raw peaks of ~0.076) sat +/// in soft-onset territory and could eat real word starts, and its tuning +/// was calibrated against `silence_scale = 0.0` audio. Deleted in favour of +/// upstream parity: accept the occasional smeared first syllable rather +/// than risk trimming real speech. +const SHORT_PROMPT_PAD_SPACES: usize = 8; + +/// sherpa-onnx's documented `frames_after_eos` default. We deliberately do +/// *not* override this knob — the previous attempt to bump it for short +/// inputs and lower it for long inputs lowered it below the upstream default +/// of 3, which clipped the leading audio of multi-clause sentences (the +/// "first 'yep' is static" regression). The constant exists only for the +/// regression test below. Source: `offline-tts-pocket-impl.h:Generate`. +#[cfg(test)] +const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; + +// ── ONNX file names (five Pocket TTS sessions plus two JSON tables) ─────────── + +const FILE_LM_MAIN: &str = "lm_main.onnx"; +const FILE_LM_FLOW: &str = "lm_flow.onnx"; +const FILE_ENCODER: &str = "encoder.onnx"; +const FILE_DECODER: &str = "decoder.onnx"; +const FILE_TEXT_COND: &str = "text_conditioner.onnx"; +const FILE_VOCAB: &str = "vocab.json"; +const FILE_TOKEN_SCORES: &str = "token_scores.json"; + +// ── Voice style ─────────────────────────────────────────────────────────────── + +/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. +/// +/// Pocket TTS takes a reference waveform per generation call (not a +/// precomputed style embedding), so we keep the samples in memory and clone +/// the small `Vec` into each `GenerationConfig` rather than re-reading the +/// WAV from disk on every sentence. +#[derive(Debug, Clone)] +pub struct VoiceStyle { + samples: Vec, + sample_rate: i32, +} + +/// Load a reference voice WAV from disk. +/// +/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS +/// resamples internally using `reference_sample_rate`. The bundled +/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. +pub fn load_voice_style(path: &Path) -> Result { + let path_str = path + .to_str() + .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; + let wave = Wave::read(path_str) + .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; + let samples = wave.samples().to_vec(); + if samples.is_empty() { + return Err(format!("voice WAV is empty: {}", path.display())); + } + Ok(VoiceStyle { + samples, + sample_rate: wave.sample_rate(), + }) +} + +// ── Engine ──────────────────────────────────────────────────────────────────── + +/// Pocket TTS engine handle. Cheap to construct (one `OfflineTts::create` +/// call). Owned by the TTS worker thread for the lifetime of a huddle session. +/// +/// `OfflineTts` does not implement `Debug`, so we don't derive it here — the +/// pipeline only needs to move the engine into the worker thread and call +/// `synth_chunk` on it, never to print it. +pub struct PocketTts { + inner: OfflineTts, +} + +/// Build the Pocket TTS engine from the model directory installed by +/// `huddle::models`. Returns `Err` if any expected ONNX or JSON file is +/// missing — readiness is normally enforced by `is_tts_ready` upstream, but +/// the check is repeated here so a manually-modified model dir produces a +/// clear error string instead of an opaque sherpa-onnx `None`. +pub fn load_text_to_speech(model_dir: &str) -> Result { + let dir = PathBuf::from(model_dir); + for name in [ + FILE_LM_MAIN, + FILE_LM_FLOW, + FILE_ENCODER, + FILE_DECODER, + FILE_TEXT_COND, + FILE_VOCAB, + FILE_TOKEN_SCORES, + ] { + let p = dir.join(name); + if !p.is_file() { + return Err(format!("missing Pocket TTS file: {}", p.display())); + } + } + + let to_str = |name: &str| -> String { dir.join(name).to_string_lossy().into_owned() }; + + // Build the config by mutating defaults — mirrors `stt.rs` and stays + // resilient if sherpa-onnx adds unrelated model-family fields. + let mut cfg = OfflineTtsConfig::default(); + cfg.model.pocket.lm_main = Some(to_str(FILE_LM_MAIN)); + cfg.model.pocket.lm_flow = Some(to_str(FILE_LM_FLOW)); + cfg.model.pocket.encoder = Some(to_str(FILE_ENCODER)); + cfg.model.pocket.decoder = Some(to_str(FILE_DECODER)); + cfg.model.pocket.text_conditioner = Some(to_str(FILE_TEXT_COND)); + cfg.model.pocket.vocab_json = Some(to_str(FILE_VOCAB)); + cfg.model.pocket.token_scores_json = Some(to_str(FILE_TOKEN_SCORES)); + cfg.model.pocket.voice_embedding_cache_capacity = VOICE_EMBEDDING_CACHE_CAPACITY; + cfg.model.num_threads = TTS_NUM_THREADS; + // Explicit — defaults are not part of the API contract, and noisy debug + // logging in release builds would be expensive on every synthesized chunk. + cfg.model.debug = false; + + let inner = OfflineTts::create(&cfg) + .ok_or_else(|| "OfflineTts::create returned None for Pocket TTS".to_string())?; + Ok(PocketTts { inner }) +} + +// ── Prompt preparation ──────────────────────────────────────────────────────── + +/// Result of [`prepare_pocket_prompt`]: a synthesizer-ready prompt plus the +/// per-call generation overrides derived from the original text. +/// +/// `None` for either override means "leave sherpa-onnx's documented default +/// in place". The pipeline only sets `max_frames` (and only for short +/// padded inputs) so it can bound the original "monster breathing" runaway +/// without disturbing the rest of the LM sampling envelope. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct PreparedPrompt { + /// Text to hand to `OfflineTts::generate_with_config`. Capitalized, + /// punctuation-terminated, and (for short inputs) left-padded with + /// spaces — upstream's mitigation for the FlowLM cold-start smear. + pub text: String, + /// Value to pass via `GenerationConfig.extra["max_frames"]`, or `None` to + /// keep the upstream default of 500 LM steps. We only override on short + /// padded prompts where we have a tight expectation on output length. + pub max_frames: Option, +} + +/// Mirror of the *text-preparation* half of upstream +/// `pocket_tts.models.tts_model.prepare_text_prompt`. Sherpa-onnx's C++ +/// Pocket TTS impl does not run these preparation steps, so short / +/// unpunctuated / lowercase inputs can trigger up to 40 s of runaway +/// generation when the EOS logit never crosses its threshold. We replicate +/// the upstream Python recipe here: +/// +/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but +/// cheap to re-check after sentence splitting). +/// 2. Capitalize the first letter. +/// 3. Append `.` if the text doesn't end in punctuation. +/// 4. If fewer than five words, prepend `SHORT_PROMPT_PAD_SPACES` spaces +/// (upstream's cold-start mitigation — see the constant's docstring) and +/// return a tight [`SHORT_PROMPT_MAX_FRAMES`] cap so the LM can't run +/// away if EOS still doesn't fire. +/// +/// We do **not** override `frames_after_eos` — sherpa-onnx's default of 3 +/// is what we want. An earlier version set it to 1 on long inputs, which +/// clipped the leading audio of multi-clause sentences ("first 'yep' is +/// just static" regression). Tests `prepare_prompt_never_lowers_frames_…` +/// lock this in. +/// +/// Returns `None` only if the input is empty after trimming — caller should +/// skip synthesis in that case. +pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { + let trimmed = input.trim(); + if trimmed.is_empty() { + return None; + } + + // Collapse stray double-spaces / embedded newlines that may slip past + // `preprocess_for_tts` when sentences are spliced back together. + let mut cleaned = String::with_capacity(trimmed.len()); + let mut last_was_space = false; + for ch in trimmed.chars() { + let is_ws = ch.is_whitespace(); + if is_ws { + if !last_was_space { + cleaned.push(' '); + } + last_was_space = true; + } else { + cleaned.push(ch); + last_was_space = false; + } + } + + // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). + let first = cleaned.chars().next().expect("cleaned non-empty above"); + if first.is_lowercase() { + let upper: String = first.to_uppercase().collect(); + let mut iter = cleaned.chars(); + iter.next(); + cleaned = upper + iter.as_str(); + } + + // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. + // The upstream Python only checks `isalnum` → period, but for our agent + // text we already may end in `!` `?` `.` etc. — treat any of those as OK. + let last = cleaned + .chars() + .next_back() + .expect("cleaned non-empty above"); + if !matches!(last, '.' | '!' | '?' | ';' | ':' | ',') { + cleaned.push('.'); + } + + // Word count of the *cleaned but not padded* text — padding is whitespace + // only and would just lie to the threshold check below. + let word_count = cleaned.split_whitespace().count(); + + let (final_text, max_frames) = if word_count <= SHORT_PROMPT_WORD_THRESHOLD { + let mut padded = String::with_capacity(cleaned.len() + SHORT_PROMPT_PAD_SPACES); + for _ in 0..SHORT_PROMPT_PAD_SPACES { + padded.push(' '); + } + padded.push_str(&cleaned); + (padded, Some(SHORT_PROMPT_MAX_FRAMES)) + } else { + // For everything ≥5 words, fall back to upstream defaults. Overriding + // these is what caused the "first 'yep' is static" regression — the + // upstream LM has been tuned for `frames_after_eos = 3` and + // `max_frames = 500`, and there's no clear win in second-guessing. + (cleaned, None) + }; + + Some(PreparedPrompt { + text: final_text, + max_frames, + }) +} + +/// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. +/// +/// Centralised so the regression test below can assert that we **never** +/// emit a `frames_after_eos` override — the previous attempt to override +/// that knob (setting it to 1 for ≥5-word inputs) clipped the leading +/// audio of multi-clause sentences (the "first 'yep' is static" bug on +/// 2026-05-18). The upstream sherpa-onnx default of 3 is what we want, and +/// the right way to keep it is to not set it at all. +fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { + prepared.max_frames.map(|mf| { + let mut h: HashMap = HashMap::with_capacity(1); + h.insert("max_frames".to_string(), serde_json::Value::from(mf)); + h + }) +} + +impl PocketTts { + /// Synthesise `text` with the given reference voice. + /// + /// `_lang` and `_steps` are accepted for API compatibility with the + /// previous Kokoro engine. Pocket TTS infers language from the input text + /// directly and is a one-step consistency model. Returns an empty buffer + /// for whitespace-only input. + pub fn synth_chunk( + &self, + text: &str, + _lang: &str, + style: &VoiceStyle, + _steps: usize, + ) -> Result, String> { + // Mirror upstream pocket-tts prompt prep — without this short or + // unpunctuated inputs can cause the LM's EOS logit to never trip, + // producing up to 40 s of "monster breathing" garbage on the first + // utterance. See `prepare_pocket_prompt` for the full recipe. + let prepared = match prepare_pocket_prompt(text) { + Some(p) => p, + None => return Ok(Vec::new()), + }; + + // Per-call generation hints sherpa-onnx forwards to + // `offline-tts-pocket-impl.h`. We only override `max_frames`, and + // only for short padded prompts where we have a tight expectation + // on output length — that bounds the original runaway without + // disturbing the rest of the LM sampling envelope. See + // `prepare_pocket_prompt` docs for the regression history. + let extra = build_generation_extra(&prepared); + + let cfg = GenerationConfig { + num_steps: SYNTH_NUM_STEPS, + silence_scale: SYNTH_SILENCE_SCALE, + reference_audio: Some(style.samples.clone()), + reference_sample_rate: style.sample_rate, + extra, + // `speed` stays at its default: the Pocket impl never reads it + // (see the engine-contract note in the module docs). + ..Default::default() + }; + + // No progress callback — synthesis is fast enough that returning the + // whole buffer at once keeps the lookahead pipelining in `tts.rs` + // simple. `None:: bool>` pins the callback type for the + // `generate_with_config` generic parameter. + let audio = self + .inner + .generate_with_config(&prepared.text, &cfg, None:: bool>) + .ok_or_else(|| { + format!( + "Pocket TTS synthesis failed for text ({} chars)", + prepared.text.len() + ) + })?; + + let sample_rate = audio.sample_rate(); + if sample_rate != SAMPLE_RATE as i32 { + eprintln!( + "buzz-desktop: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ + (expected {SAMPLE_RATE}Hz); playback speed may be wrong" + ); + } + + Ok(audio.samples().to_vec()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── prepare_pocket_prompt ──────────────────────────────────────────────── + + #[test] + fn prepare_prompt_returns_none_for_empty_input() { + assert!(prepare_pocket_prompt("").is_none()); + assert!(prepare_pocket_prompt(" ").is_none()); + assert!(prepare_pocket_prompt("\n\t ").is_none()); + } + + /// Helper: the exact leading sequence prepended to every short prompt — + /// 8 spaces of padding (upstream's cold-start mitigation). + /// Centralising this keeps the assertions readable. + fn short_prefix() -> String { + " ".repeat(SHORT_PROMPT_PAD_SPACES) + } + + #[test] + fn prepare_prompt_pads_and_capitalizes_one_word() { + // The "yep" case Tyler hit in production — bare lowercase one-word + // utterance with no punctuation. Must be padded with the short-prompt + // space pad, capitalized, terminated, with a tight `max_frames` cap + // to bound runaway gen. + let out = prepare_pocket_prompt("yep").expect("non-empty"); + assert_eq!(out.text, format!("{}Yep.", short_prefix())); + assert_eq!(out.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); + const { + assert!( + SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT, + "short cap must be tighter than the upstream default" + ); + } + } + + #[test] + fn prepare_prompt_preserves_existing_punctuation() { + let out = prepare_pocket_prompt("yes!").expect("non-empty"); + assert_eq!(out.text, format!("{}Yes!", short_prefix())); // exclamation kept + let out = prepare_pocket_prompt("really?").expect("non-empty"); + assert_eq!(out.text, format!("{}Really?", short_prefix())); + } + + #[test] + fn prepare_prompt_threshold_is_inclusive_at_four_words() { + // 4 words = short (padded + tight max_frames); 5 words = long + // (no padding, no overrides — upstream defaults stand). + let four = prepare_pocket_prompt("one two three four").expect("non-empty"); + assert_eq!( + four.text, + format!("{}One two three four.", short_prefix()), + "four-word input should get exactly the space pad" + ); + assert_eq!(four.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); + + let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); + assert!( + !five.text.starts_with(' '), + "five-word input should NOT be padded" + ); + assert_eq!( + five.max_frames, None, + "long inputs must leave sherpa-onnx's max_frames default in place" + ); + } + + #[test] + fn prepare_prompt_does_not_pad_long_text() { + let long = "This is a longer sentence that the model should handle just fine."; + let out = prepare_pocket_prompt(long).expect("non-empty"); + assert!(!out.text.starts_with(' ')); + assert_eq!(out.max_frames, None); + assert!(out.text.ends_with('.')); + } + + #[test] + fn prepare_prompt_collapses_whitespace() { + let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); + // 3 words → short → padded. Interior whitespace collapsed. + assert_eq!(out.text, format!("{}Hello world friend.", short_prefix())); + } + + #[test] + fn prepare_prompt_does_not_double_capitalize_already_uppercase() { + let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); + assert_eq!(out.text, format!("{}HELLO there.", short_prefix())); + } + + #[test] + fn prepare_prompt_handles_non_ascii_first_letter() { + // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce + // mojibake. + let out = prepare_pocket_prompt("дa").expect("non-empty"); + assert!(out.text.contains("Дa.")); + } + + /// REGRESSION GUARD: short prompts must receive *only* whitespace + /// padding — no sacrificial text. A previous revision prepended a + /// `". . "` cold-start absorber and trimmed the rendered audio back out + /// with an amplitude threshold that could eat soft word onsets. If + /// non-whitespace ever reappears in the pad, the synth output will + /// contain audio for text the user never wrote. + #[test] + fn prepare_prompt_pad_is_whitespace_only() { + let out = prepare_pocket_prompt("I'm happy.").expect("non-empty"); + let pad_len = out.text.len() - "I'm happy.".len(); + assert!( + out.text[..pad_len].chars().all(|c| c == ' '), + "short-prompt pad must be spaces only, got {:?}", + &out.text[..pad_len] + ); + assert_eq!(out.text, format!("{}I'm happy.", short_prefix())); + } + + // ── build_generation_extra ─────────────────────────────────────────────── + // + // These tests pin down a behaviour we've now regressed twice on: + // 1) Not padding/punctuating short inputs → 40 s of "monster breathing" + // (pre-773a2a1). + // 2) Setting `frames_after_eos = 1` on long inputs → clipped leading + // audio of multi-clause sentences, e.g. "Yep, I can hear you. …" + // came out as a static burst (the 773a2a1 regression Tyler hit on + // 2026-05-18 ~14:30 UTC). + // + // The contract we enforce going forward: we **only** override + // `max_frames`, and only for ≤4-word inputs. Every other knob is left + // at sherpa-onnx's documented default (notably `frames_after_eos = 3`). + + #[test] + fn build_extra_short_prompt_sets_only_max_frames() { + let prepared = prepare_pocket_prompt("yep").expect("non-empty"); + let extra = build_generation_extra(&prepared).expect("short prompts get extra"); + // Exactly one key — `max_frames` — and nothing else. + assert_eq!(extra.len(), 1, "extra has unexpected keys: {extra:?}"); + assert_eq!( + extra.get("max_frames"), + Some(&serde_json::Value::from(SHORT_PROMPT_MAX_FRAMES)) + ); + assert!( + !extra.contains_key("frames_after_eos"), + "frames_after_eos must never be set — upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT} is what we want" + ); + } + + #[test] + fn build_extra_long_prompt_is_none() { + // ≥5 words: no extras at all. This is the key fix for the "first + // 'yep' in 'Yep, I can hear you. …' is static" regression — we + // were previously forcing `frames_after_eos = 1` on this path. + let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); + assert_eq!( + build_generation_extra(&prepared), + None, + "long prompts must not override any LM knob" + ); + } + + #[test] + fn build_extra_never_lowers_frames_after_eos_for_any_word_count() { + // Sweep a range of prompt lengths and assert the `extra` map (when + // present) never carries a `frames_after_eos` override that's lower + // than the upstream sherpa-onnx default. Implemented as a structural + // check — we just never set the key — but worth a property test in + // case someone reintroduces the override in the future. + let prompts: &[&str] = &[ + "hi", + "hi there", + "yes please", + "one two three four", + "one two three four five", + "a slightly longer reply, hopefully fine", + "This is a multi-clause sentence. It has two parts.", + "really really really really really long prompt with lots of words just to be sure", + ]; + for &p in prompts { + let prepared = prepare_pocket_prompt(p).expect("non-empty"); + if let Some(extra) = build_generation_extra(&prepared) { + if let Some(v) = extra.get("frames_after_eos") { + let n = v.as_i64().expect("frames_after_eos should be int"); + assert!( + n >= SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT as i64, + "prompt {p:?} set frames_after_eos={n}, below upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT}" + ); + } + } + } + } + + #[test] + fn short_prompt_max_frames_is_below_upstream_default() { + // Sanity: the override only ever *lowers* the cap, never raises it. + const { + assert!(SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT); + } + // …and is still large enough for a one-to-four-word reply. At Mimi's + // 12.5 Hz frame rate, 100 frames = 8 s, which is roomy. + const { + assert!(SHORT_PROMPT_MAX_FRAMES >= 50, "would risk truncation"); + } + } +} diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 66553ef595..257707947e 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1023,6 +1023,7 @@ dependencies = [ "buzz-media", "buzz-persona", "buzz-sdk", + "buzz-voice", "bytes", "bzip2 0.6.1", "chrono", @@ -1143,6 +1144,14 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "serde_json", + "sherpa-onnx", +] + [[package]] name = "by_address" version = "1.2.1" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 324218a49d..1cafc80f72 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -89,6 +89,7 @@ buzz_core_pkg = { package = "buzz-core", path = "../../crates/buzz-core" } buzz_persona_pkg = { package = "buzz-persona", path = "../../crates/buzz-persona" } buzz_sdk_pkg = { package = "buzz-sdk", path = "../../crates/buzz-sdk" } buzz_agent_pkg = { package = "buzz-agent", path = "../../crates/buzz-agent" } +buzz_voice_pkg = { package = "buzz-voice", path = "../../crates/buzz-voice" } iroh = { version = "1.0.2", optional = true } mesh-llm-sdk = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-sdk", default-features = false, features = ["client", "serving"], optional = true } mesh-llm-host-runtime = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-host-runtime", default-features = false, features = ["dynamic-native-runtime"], optional = true } diff --git a/desktop/src-tauri/examples/pocket_quality_ab.rs b/desktop/src-tauri/examples/pocket_quality_ab.rs index 0c31f1c910..95f2e4815f 100644 --- a/desktop/src-tauri/examples/pocket_quality_ab.rs +++ b/desktop/src-tauri/examples/pocket_quality_ab.rs @@ -19,7 +19,7 @@ // standalone corpus generator deliberately does not call. #![allow(dead_code)] -#[path = "../src/huddle/pocket.rs"] +#[path = "../../../crates/buzz-voice/src/pocket.rs"] mod production_pocket; #[path = "../src/huddle/preprocessing.rs"] mod production_preprocessing; diff --git a/desktop/src-tauri/src/huddle/pocket.rs b/desktop/src-tauri/src/huddle/pocket.rs index ee1faf928a..8e19781eeb 100644 --- a/desktop/src-tauri/src/huddle/pocket.rs +++ b/desktop/src-tauri/src/huddle/pocket.rs @@ -1,654 +1,3 @@ -//! Pocket TTS engine wrapper around sherpa-onnx's `OfflineTts`. -//! -//! Pocket TTS is a small (~473 MB fp32 ONNX) zero-shot voice-cloning TTS -//! model from Kyutai. It runs quickly on CPU via sherpa-onnx, replacing the -//! previous Kokoro-82M engine that also required an espeak-free but -//! lexicon-heavy G2P pipeline (Misaki + CMUdict). -//! -//! Full-precision fp32 sessions, not the ~189 MB int8 quantization we -//! originally shipped: a direct same-runtime A/B (k2-fsa/sherpa-onnx#3172) -//! found the int8 ONNX export audibly degraded output quality, and fp32 -//! "significantly improved quality even at 1 step". -//! -//! ## Attribution -//! -//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. -//! arXiv:2509.06926. Original repository: . -//! Licensed CC-BY-4.0. -//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. -//! - **ONNX export**: KevinAHM — -//! . CC-BY-4.0. -//! - **sherpa-onnx repackage**: csukuangfj / k2-fsa — -//! . -//! Repackages KevinAHM's export with the file layout sherpa-onnx's -//! `OfflineTtsPocketModelConfig` expects. CC-BY-4.0. -//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary -//! (f, conversation)" preset from the Kyutai TTS demo -//! (), which maps to `vctk/p333_023_enhanced.wav` -//! in . CC-BY-4.0, base recording -//! from the VCTK corpus, enhanced by ai-coustics. -//! -//! Buzz ships these files unmodified; see the on-disk `MODEL_LICENSE.txt` -//! sidecar written by `huddle::models` during install for the canonical -//! CC-BY-4.0 §3(a)(1) attribution block. -//! -//! ## Engine-module contract (see `huddle::tts`) -//! -//! `pocket.rs` exposes a fixed surface used by `tts.rs`. Mirroring this -//! contract is what lets the TTS pipeline stay engine-agnostic: -//! -//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. -//! - `DEFAULT_VOICE: &str` — default voice name (without extension). -//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. -//! - `load_text_to_speech(model_dir)` → `Result` -//! - `load_voice_style(path)` → `Result` -//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` -//! → `Result, String>` -//! -//! `lang` and `steps` are accepted for API compatibility with the previous -//! Kokoro engine but are unused — Pocket TTS does its own language ID from -//! the input text and is not a diffusion model (consistency LM, one step). -//! There is no speed knob: sherpa-onnx's `GenerationConfig.speed` is only -//! read by some model families (vits), never by the Pocket impl -//! (`offline-tts-pocket-impl.h` — zero references), and upstream pocket-tts -//! has no speed parameter either. +//! Compatibility re-export for the shared Pocket TTS engine. -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; - -// ── Engine-module contract: public consts ───────────────────────────────────── - -/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, -/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. -pub const SAMPLE_RATE: u32 = 24_000; - -/// Name (without extension) of the bundled reference voice. The model directory -/// is expected to contain `.` after install. -pub const DEFAULT_VOICE: &str = "reference_sample"; - -/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the -/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, -/// not a precomputed embedding, so the extension change is honest. -pub const VOICE_FILE_EXT: &str = "wav"; - -// ── Tuning ──────────────────────────────────────────────────────────────────── - -/// Single-threaded ONNX execution for predictable CPU contention with the STT -/// pipeline. Matches `STT_NUM_THREADS` in `stt.rs`; raise only if a benchmark -/// argues for it. -const TTS_NUM_THREADS: i32 = 1; - -/// LRU cache size for cloned voice embeddings inside the sherpa-onnx engine. -/// We bind to one voice per pipeline today, but the upstream example uses 16 -/// and the cost is negligible — keep room for future multi-voice support. -const VOICE_EMBEDDING_CACHE_CAPACITY: i32 = 16; - -/// Pocket TTS is a consistency-based LM. Generation quality saturates at one -/// denoising step — the upstream `GenerationConfig` default of 5 multiplies -/// synthesis time by ~5× with no audible benefit on this model. -const SYNTH_NUM_STEPS: i32 = 1; - -/// Leave the generated audio's silences untouched (1.0 is the identity). -/// -/// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding -/// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and -/// multiplies its length by this factor. The previous value of 0.0 — set -/// under the mistaken belief it disabled lead-in/lead-out padding — deleted -/// every natural pause inside an utterance: clause breaks, breaths, the gap -/// after a comma. Words slammed together and endings cut abruptly. The -/// reference Pocket TTS pipeline does not post-process silence at all; -/// 1.0 restores parity. -const SYNTH_SILENCE_SCALE: f32 = 1.0; - -/// sherpa-onnx upstream default for `max_frames` (LM steps), in -/// `offline-tts-pocket-impl.h:Generate`. 500 steps ≈ 40 s of audio at the -/// Mimi 12.5 Hz frame rate. Referenced only by the regression test below; -/// production code path never raises (or even reads) this value — we just -/// leave sherpa-onnx's own default in place by not setting the override. -#[cfg(test)] -const SHERPA_ONNX_MAX_FRAMES_DEFAULT: i32 = 500; - -/// Tight `max_frames` we ask for on short, padded prompts to bound the -/// original "monster breathing" runaway. 100 LM steps ≈ 8 s of audio — -/// roomy for any one-to-four-word utterance the user is likely to elicit -/// while still well short of the 40 s upstream default. Chosen with slack so -/// we never *truncate* a legitimate short reply. -const SHORT_PROMPT_MAX_FRAMES: i32 = 100; - -/// Word-count threshold (inclusive) below which we pad the prompt with -/// leading spaces and cap `max_frames` tighter than the upstream default. -/// Matches upstream `pocket_tts.models.tts_model.prepare_text_prompt`. Above -/// this threshold we leave sherpa-onnx's own defaults in place — overriding -/// them caused the "first 'yep' is just static" regression seen on -/// 2026-05-18, where dropping `frames_after_eos` below the upstream default -/// of 3 clipped the leading audio of multi-clause sentences. -const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; - -/// Number of leading spaces prepended to short prompts. The upstream Python -/// uses exactly 8 — keep parity rather than tuning blindly. -/// -/// This is upstream's *only* mitigation for the FlowLM cold-start smear on -/// short utterances (kyutai-labs/pocket-tts #91, #70): the autoregressive -/// generation has a 2–3 step "settle" period where the first phoneme can be -/// smeared. A previous revision added a sacrificial `". . "` prefix plus an -/// amplitude-threshold trim to strip the rendered prefix from the output — -/// but the trim's absolute threshold (0.02 against raw peaks of ~0.076) sat -/// in soft-onset territory and could eat real word starts, and its tuning -/// was calibrated against `silence_scale = 0.0` audio. Deleted in favour of -/// upstream parity: accept the occasional smeared first syllable rather -/// than risk trimming real speech. -const SHORT_PROMPT_PAD_SPACES: usize = 8; - -/// sherpa-onnx's documented `frames_after_eos` default. We deliberately do -/// *not* override this knob — the previous attempt to bump it for short -/// inputs and lower it for long inputs lowered it below the upstream default -/// of 3, which clipped the leading audio of multi-clause sentences (the -/// "first 'yep' is static" regression). The constant exists only for the -/// regression test below. Source: `offline-tts-pocket-impl.h:Generate`. -#[cfg(test)] -const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; - -// ── ONNX file names (five Pocket TTS sessions plus two JSON tables) ─────────── - -const FILE_LM_MAIN: &str = "lm_main.onnx"; -const FILE_LM_FLOW: &str = "lm_flow.onnx"; -const FILE_ENCODER: &str = "encoder.onnx"; -const FILE_DECODER: &str = "decoder.onnx"; -const FILE_TEXT_COND: &str = "text_conditioner.onnx"; -const FILE_VOCAB: &str = "vocab.json"; -const FILE_TOKEN_SCORES: &str = "token_scores.json"; - -// ── Voice style ─────────────────────────────────────────────────────────────── - -/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. -/// -/// Pocket TTS takes a reference waveform per generation call (not a -/// precomputed style embedding), so we keep the samples in memory and clone -/// the small `Vec` into each `GenerationConfig` rather than re-reading the -/// WAV from disk on every sentence. -#[derive(Debug, Clone)] -pub struct VoiceStyle { - samples: Vec, - sample_rate: i32, -} - -/// Load a reference voice WAV from disk. -/// -/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS -/// resamples internally using `reference_sample_rate`. The bundled -/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. -pub fn load_voice_style(path: &Path) -> Result { - let path_str = path - .to_str() - .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; - let wave = Wave::read(path_str) - .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; - let samples = wave.samples().to_vec(); - if samples.is_empty() { - return Err(format!("voice WAV is empty: {}", path.display())); - } - Ok(VoiceStyle { - samples, - sample_rate: wave.sample_rate(), - }) -} - -// ── Engine ──────────────────────────────────────────────────────────────────── - -/// Pocket TTS engine handle. Cheap to construct (one `OfflineTts::create` -/// call). Owned by the TTS worker thread for the lifetime of a huddle session. -/// -/// `OfflineTts` does not implement `Debug`, so we don't derive it here — the -/// pipeline only needs to move the engine into the worker thread and call -/// `synth_chunk` on it, never to print it. -pub struct PocketTts { - inner: OfflineTts, -} - -/// Build the Pocket TTS engine from the model directory installed by -/// `huddle::models`. Returns `Err` if any expected ONNX or JSON file is -/// missing — readiness is normally enforced by `is_tts_ready` upstream, but -/// the check is repeated here so a manually-modified model dir produces a -/// clear error string instead of an opaque sherpa-onnx `None`. -pub fn load_text_to_speech(model_dir: &str) -> Result { - let dir = PathBuf::from(model_dir); - for name in [ - FILE_LM_MAIN, - FILE_LM_FLOW, - FILE_ENCODER, - FILE_DECODER, - FILE_TEXT_COND, - FILE_VOCAB, - FILE_TOKEN_SCORES, - ] { - let p = dir.join(name); - if !p.is_file() { - return Err(format!("missing Pocket TTS file: {}", p.display())); - } - } - - let to_str = |name: &str| -> String { dir.join(name).to_string_lossy().into_owned() }; - - // Build the config by mutating defaults — mirrors `stt.rs` and stays - // resilient if sherpa-onnx adds unrelated model-family fields. - let mut cfg = OfflineTtsConfig::default(); - cfg.model.pocket.lm_main = Some(to_str(FILE_LM_MAIN)); - cfg.model.pocket.lm_flow = Some(to_str(FILE_LM_FLOW)); - cfg.model.pocket.encoder = Some(to_str(FILE_ENCODER)); - cfg.model.pocket.decoder = Some(to_str(FILE_DECODER)); - cfg.model.pocket.text_conditioner = Some(to_str(FILE_TEXT_COND)); - cfg.model.pocket.vocab_json = Some(to_str(FILE_VOCAB)); - cfg.model.pocket.token_scores_json = Some(to_str(FILE_TOKEN_SCORES)); - cfg.model.pocket.voice_embedding_cache_capacity = VOICE_EMBEDDING_CACHE_CAPACITY; - cfg.model.num_threads = TTS_NUM_THREADS; - // Explicit — defaults are not part of the API contract, and noisy debug - // logging in release builds would be expensive on every synthesized chunk. - cfg.model.debug = false; - - let inner = OfflineTts::create(&cfg) - .ok_or_else(|| "OfflineTts::create returned None for Pocket TTS".to_string())?; - Ok(PocketTts { inner }) -} - -// ── Prompt preparation ──────────────────────────────────────────────────────── - -/// Result of [`prepare_pocket_prompt`]: a synthesizer-ready prompt plus the -/// per-call generation overrides derived from the original text. -/// -/// `None` for either override means "leave sherpa-onnx's documented default -/// in place". The pipeline only sets `max_frames` (and only for short -/// padded inputs) so it can bound the original "monster breathing" runaway -/// without disturbing the rest of the LM sampling envelope. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct PreparedPrompt { - /// Text to hand to `OfflineTts::generate_with_config`. Capitalized, - /// punctuation-terminated, and (for short inputs) left-padded with - /// spaces — upstream's mitigation for the FlowLM cold-start smear. - pub text: String, - /// Value to pass via `GenerationConfig.extra["max_frames"]`, or `None` to - /// keep the upstream default of 500 LM steps. We only override on short - /// padded prompts where we have a tight expectation on output length. - pub max_frames: Option, -} - -/// Mirror of the *text-preparation* half of upstream -/// `pocket_tts.models.tts_model.prepare_text_prompt`. Sherpa-onnx's C++ -/// Pocket TTS impl does not run these preparation steps, so short / -/// unpunctuated / lowercase inputs can trigger up to 40 s of runaway -/// generation when the EOS logit never crosses its threshold. We replicate -/// the upstream Python recipe here: -/// -/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but -/// cheap to re-check after sentence splitting). -/// 2. Capitalize the first letter. -/// 3. Append `.` if the text doesn't end in punctuation. -/// 4. If fewer than five words, prepend `SHORT_PROMPT_PAD_SPACES` spaces -/// (upstream's cold-start mitigation — see the constant's docstring) and -/// return a tight [`SHORT_PROMPT_MAX_FRAMES`] cap so the LM can't run -/// away if EOS still doesn't fire. -/// -/// We do **not** override `frames_after_eos` — sherpa-onnx's default of 3 -/// is what we want. An earlier version set it to 1 on long inputs, which -/// clipped the leading audio of multi-clause sentences ("first 'yep' is -/// just static" regression). Tests `prepare_prompt_never_lowers_frames_…` -/// lock this in. -/// -/// Returns `None` only if the input is empty after trimming — caller should -/// skip synthesis in that case. -pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { - let trimmed = input.trim(); - if trimmed.is_empty() { - return None; - } - - // Collapse stray double-spaces / embedded newlines that may slip past - // `preprocess_for_tts` when sentences are spliced back together. - let mut cleaned = String::with_capacity(trimmed.len()); - let mut last_was_space = false; - for ch in trimmed.chars() { - let is_ws = ch.is_whitespace(); - if is_ws { - if !last_was_space { - cleaned.push(' '); - } - last_was_space = true; - } else { - cleaned.push(ch); - last_was_space = false; - } - } - - // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). - let first = cleaned.chars().next().expect("cleaned non-empty above"); - if first.is_lowercase() { - let upper: String = first.to_uppercase().collect(); - let mut iter = cleaned.chars(); - iter.next(); - cleaned = upper + iter.as_str(); - } - - // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. - // The upstream Python only checks `isalnum` → period, but for our agent - // text we already may end in `!` `?` `.` etc. — treat any of those as OK. - let last = cleaned - .chars() - .next_back() - .expect("cleaned non-empty above"); - if !matches!(last, '.' | '!' | '?' | ';' | ':' | ',') { - cleaned.push('.'); - } - - // Word count of the *cleaned but not padded* text — padding is whitespace - // only and would just lie to the threshold check below. - let word_count = cleaned.split_whitespace().count(); - - let (final_text, max_frames) = if word_count <= SHORT_PROMPT_WORD_THRESHOLD { - let mut padded = String::with_capacity(cleaned.len() + SHORT_PROMPT_PAD_SPACES); - for _ in 0..SHORT_PROMPT_PAD_SPACES { - padded.push(' '); - } - padded.push_str(&cleaned); - (padded, Some(SHORT_PROMPT_MAX_FRAMES)) - } else { - // For everything ≥5 words, fall back to upstream defaults. Overriding - // these is what caused the "first 'yep' is static" regression — the - // upstream LM has been tuned for `frames_after_eos = 3` and - // `max_frames = 500`, and there's no clear win in second-guessing. - (cleaned, None) - }; - - Some(PreparedPrompt { - text: final_text, - max_frames, - }) -} - -/// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. -/// -/// Centralised so the regression test below can assert that we **never** -/// emit a `frames_after_eos` override — the previous attempt to override -/// that knob (setting it to 1 for ≥5-word inputs) clipped the leading -/// audio of multi-clause sentences (the "first 'yep' is static" bug on -/// 2026-05-18). The upstream sherpa-onnx default of 3 is what we want, and -/// the right way to keep it is to not set it at all. -fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { - prepared.max_frames.map(|mf| { - let mut h: HashMap = HashMap::with_capacity(1); - h.insert("max_frames".to_string(), serde_json::Value::from(mf)); - h - }) -} - -impl PocketTts { - /// Synthesise `text` with the given reference voice. - /// - /// `_lang` and `_steps` are accepted for API compatibility with the - /// previous Kokoro engine. Pocket TTS infers language from the input text - /// directly and is a one-step consistency model. Returns an empty buffer - /// for whitespace-only input. - pub fn synth_chunk( - &self, - text: &str, - _lang: &str, - style: &VoiceStyle, - _steps: usize, - ) -> Result, String> { - // Mirror upstream pocket-tts prompt prep — without this short or - // unpunctuated inputs can cause the LM's EOS logit to never trip, - // producing up to 40 s of "monster breathing" garbage on the first - // utterance. See `prepare_pocket_prompt` for the full recipe. - let prepared = match prepare_pocket_prompt(text) { - Some(p) => p, - None => return Ok(Vec::new()), - }; - - // Per-call generation hints sherpa-onnx forwards to - // `offline-tts-pocket-impl.h`. We only override `max_frames`, and - // only for short padded prompts where we have a tight expectation - // on output length — that bounds the original runaway without - // disturbing the rest of the LM sampling envelope. See - // `prepare_pocket_prompt` docs for the regression history. - let extra = build_generation_extra(&prepared); - - let cfg = GenerationConfig { - num_steps: SYNTH_NUM_STEPS, - silence_scale: SYNTH_SILENCE_SCALE, - reference_audio: Some(style.samples.clone()), - reference_sample_rate: style.sample_rate, - extra, - // `speed` stays at its default: the Pocket impl never reads it - // (see the engine-contract note in the module docs). - ..Default::default() - }; - - // No progress callback — synthesis is fast enough that returning the - // whole buffer at once keeps the lookahead pipelining in `tts.rs` - // simple. `None:: bool>` pins the callback type for the - // `generate_with_config` generic parameter. - let audio = self - .inner - .generate_with_config(&prepared.text, &cfg, None:: bool>) - .ok_or_else(|| { - format!( - "Pocket TTS synthesis failed for text ({} chars)", - prepared.text.len() - ) - })?; - - let sample_rate = audio.sample_rate(); - if sample_rate != SAMPLE_RATE as i32 { - eprintln!( - "buzz-desktop: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ - (expected {SAMPLE_RATE}Hz); playback speed may be wrong" - ); - } - - Ok(audio.samples().to_vec()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // ── prepare_pocket_prompt ──────────────────────────────────────────────── - - #[test] - fn prepare_prompt_returns_none_for_empty_input() { - assert!(prepare_pocket_prompt("").is_none()); - assert!(prepare_pocket_prompt(" ").is_none()); - assert!(prepare_pocket_prompt("\n\t ").is_none()); - } - - /// Helper: the exact leading sequence prepended to every short prompt — - /// 8 spaces of padding (upstream's cold-start mitigation). - /// Centralising this keeps the assertions readable. - fn short_prefix() -> String { - " ".repeat(SHORT_PROMPT_PAD_SPACES) - } - - #[test] - fn prepare_prompt_pads_and_capitalizes_one_word() { - // The "yep" case Tyler hit in production — bare lowercase one-word - // utterance with no punctuation. Must be padded with the short-prompt - // space pad, capitalized, terminated, with a tight `max_frames` cap - // to bound runaway gen. - let out = prepare_pocket_prompt("yep").expect("non-empty"); - assert_eq!(out.text, format!("{}Yep.", short_prefix())); - assert_eq!(out.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - const { - assert!( - SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT, - "short cap must be tighter than the upstream default" - ); - } - } - - #[test] - fn prepare_prompt_preserves_existing_punctuation() { - let out = prepare_pocket_prompt("yes!").expect("non-empty"); - assert_eq!(out.text, format!("{}Yes!", short_prefix())); // exclamation kept - let out = prepare_pocket_prompt("really?").expect("non-empty"); - assert_eq!(out.text, format!("{}Really?", short_prefix())); - } - - #[test] - fn prepare_prompt_threshold_is_inclusive_at_four_words() { - // 4 words = short (padded + tight max_frames); 5 words = long - // (no padding, no overrides — upstream defaults stand). - let four = prepare_pocket_prompt("one two three four").expect("non-empty"); - assert_eq!( - four.text, - format!("{}One two three four.", short_prefix()), - "four-word input should get exactly the space pad" - ); - assert_eq!(four.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - - let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); - assert!( - !five.text.starts_with(' '), - "five-word input should NOT be padded" - ); - assert_eq!( - five.max_frames, None, - "long inputs must leave sherpa-onnx's max_frames default in place" - ); - } - - #[test] - fn prepare_prompt_does_not_pad_long_text() { - let long = "This is a longer sentence that the model should handle just fine."; - let out = prepare_pocket_prompt(long).expect("non-empty"); - assert!(!out.text.starts_with(' ')); - assert_eq!(out.max_frames, None); - assert!(out.text.ends_with('.')); - } - - #[test] - fn prepare_prompt_collapses_whitespace() { - let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); - // 3 words → short → padded. Interior whitespace collapsed. - assert_eq!(out.text, format!("{}Hello world friend.", short_prefix())); - } - - #[test] - fn prepare_prompt_does_not_double_capitalize_already_uppercase() { - let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); - assert_eq!(out.text, format!("{}HELLO there.", short_prefix())); - } - - #[test] - fn prepare_prompt_handles_non_ascii_first_letter() { - // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce - // mojibake. - let out = prepare_pocket_prompt("дa").expect("non-empty"); - assert!(out.text.contains("Дa.")); - } - - /// REGRESSION GUARD: short prompts must receive *only* whitespace - /// padding — no sacrificial text. A previous revision prepended a - /// `". . "` cold-start absorber and trimmed the rendered audio back out - /// with an amplitude threshold that could eat soft word onsets. If - /// non-whitespace ever reappears in the pad, the synth output will - /// contain audio for text the user never wrote. - #[test] - fn prepare_prompt_pad_is_whitespace_only() { - let out = prepare_pocket_prompt("I'm happy.").expect("non-empty"); - let pad_len = out.text.len() - "I'm happy.".len(); - assert!( - out.text[..pad_len].chars().all(|c| c == ' '), - "short-prompt pad must be spaces only, got {:?}", - &out.text[..pad_len] - ); - assert_eq!(out.text, format!("{}I'm happy.", short_prefix())); - } - - // ── build_generation_extra ─────────────────────────────────────────────── - // - // These tests pin down a behaviour we've now regressed twice on: - // 1) Not padding/punctuating short inputs → 40 s of "monster breathing" - // (pre-773a2a1). - // 2) Setting `frames_after_eos = 1` on long inputs → clipped leading - // audio of multi-clause sentences, e.g. "Yep, I can hear you. …" - // came out as a static burst (the 773a2a1 regression Tyler hit on - // 2026-05-18 ~14:30 UTC). - // - // The contract we enforce going forward: we **only** override - // `max_frames`, and only for ≤4-word inputs. Every other knob is left - // at sherpa-onnx's documented default (notably `frames_after_eos = 3`). - - #[test] - fn build_extra_short_prompt_sets_only_max_frames() { - let prepared = prepare_pocket_prompt("yep").expect("non-empty"); - let extra = build_generation_extra(&prepared).expect("short prompts get extra"); - // Exactly one key — `max_frames` — and nothing else. - assert_eq!(extra.len(), 1, "extra has unexpected keys: {extra:?}"); - assert_eq!( - extra.get("max_frames"), - Some(&serde_json::Value::from(SHORT_PROMPT_MAX_FRAMES)) - ); - assert!( - !extra.contains_key("frames_after_eos"), - "frames_after_eos must never be set — upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT} is what we want" - ); - } - - #[test] - fn build_extra_long_prompt_is_none() { - // ≥5 words: no extras at all. This is the key fix for the "first - // 'yep' in 'Yep, I can hear you. …' is static" regression — we - // were previously forcing `frames_after_eos = 1` on this path. - let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); - assert_eq!( - build_generation_extra(&prepared), - None, - "long prompts must not override any LM knob" - ); - } - - #[test] - fn build_extra_never_lowers_frames_after_eos_for_any_word_count() { - // Sweep a range of prompt lengths and assert the `extra` map (when - // present) never carries a `frames_after_eos` override that's lower - // than the upstream sherpa-onnx default. Implemented as a structural - // check — we just never set the key — but worth a property test in - // case someone reintroduces the override in the future. - let prompts: &[&str] = &[ - "hi", - "hi there", - "yes please", - "one two three four", - "one two three four five", - "a slightly longer reply, hopefully fine", - "This is a multi-clause sentence. It has two parts.", - "really really really really really long prompt with lots of words just to be sure", - ]; - for &p in prompts { - let prepared = prepare_pocket_prompt(p).expect("non-empty"); - if let Some(extra) = build_generation_extra(&prepared) { - if let Some(v) = extra.get("frames_after_eos") { - let n = v.as_i64().expect("frames_after_eos should be int"); - assert!( - n >= SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT as i64, - "prompt {p:?} set frames_after_eos={n}, below upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT}" - ); - } - } - } - } - - #[test] - fn short_prompt_max_frames_is_below_upstream_default() { - // Sanity: the override only ever *lowers* the cap, never raises it. - const { - assert!(SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT); - } - // …and is still large enough for a one-to-four-word reply. At Mimi's - // 12.5 Hz frame rate, 100 frames = 8 s, which is roomy. - const { - assert!(SHORT_PROMPT_MAX_FRAMES >= 50, "would risk truncation"); - } - } -} +pub use buzz_voice_pkg::pocket::*; diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 4dd1142801..ae26fc6ba8 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -84,6 +84,9 @@ run_unit_tests() { run_test_step "buzz-auth unit tests" \ cargo test -p buzz-auth --lib -- --nocapture + run_test_step "buzz-voice tests" \ + cargo test -p buzz-voice --lib -- --nocapture + # buzz-db migrator/lint unit tests (no infra): guard the embedded-migrator # invariant (exactly the consolidated 0001; cutover/backfill stays an operator # script, not startup state) and the tenant-scoping lints. The Postgres-backed From 6bbfb6db6570a8c9bbe27cf58aa3bf0ffa7f43da Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 27 Jul 2026 23:51:07 -0400 Subject: [PATCH 2/8] Make Pocket TTS comments timeless Signed-off-by: John Tennant --- crates/buzz-voice/src/pocket.rs | 71 ++++++++++----------------------- 1 file changed, 22 insertions(+), 49 deletions(-) diff --git a/crates/buzz-voice/src/pocket.rs b/crates/buzz-voice/src/pocket.rs index ee1faf928a..e7daa3ed96 100644 --- a/crates/buzz-voice/src/pocket.rs +++ b/crates/buzz-voice/src/pocket.rs @@ -1,14 +1,11 @@ //! Pocket TTS engine wrapper around sherpa-onnx's `OfflineTts`. //! //! Pocket TTS is a small (~473 MB fp32 ONNX) zero-shot voice-cloning TTS -//! model from Kyutai. It runs quickly on CPU via sherpa-onnx, replacing the -//! previous Kokoro-82M engine that also required an espeak-free but -//! lexicon-heavy G2P pipeline (Misaki + CMUdict). +//! model from Kyutai that runs quickly on CPU via sherpa-onnx. //! -//! Full-precision fp32 sessions, not the ~189 MB int8 quantization we -//! originally shipped: a direct same-runtime A/B (k2-fsa/sherpa-onnx#3172) -//! found the int8 ONNX export audibly degraded output quality, and fp32 -//! "significantly improved quality even at 1 step". +//! Buzz uses full-precision fp32 sessions because a direct same-runtime A/B +//! (k2-fsa/sherpa-onnx#3172) found the ~189 MB int8 ONNX export audibly +//! degraded output quality. //! //! ## Attribution //! @@ -94,12 +91,9 @@ const SYNTH_NUM_STEPS: i32 = 1; /// /// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding /// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and -/// multiplies its length by this factor. The previous value of 0.0 — set -/// under the mistaken belief it disabled lead-in/lead-out padding — deleted -/// every natural pause inside an utterance: clause breaks, breaths, the gap -/// after a comma. Words slammed together and endings cut abruptly. The -/// reference Pocket TTS pipeline does not post-process silence at all; -/// 1.0 restores parity. +/// multiplies its length by this factor. The reference Pocket TTS pipeline +/// preserves natural clause breaks, breaths, and punctuation pauses, so the +/// identity scale keeps those interior silences intact. const SYNTH_SILENCE_SCALE: f32 = 1.0; /// sherpa-onnx upstream default for `max_frames` (LM steps), in @@ -120,10 +114,9 @@ const SHORT_PROMPT_MAX_FRAMES: i32 = 100; /// Word-count threshold (inclusive) below which we pad the prompt with /// leading spaces and cap `max_frames` tighter than the upstream default. /// Matches upstream `pocket_tts.models.tts_model.prepare_text_prompt`. Above -/// this threshold we leave sherpa-onnx's own defaults in place — overriding -/// them caused the "first 'yep' is just static" regression seen on -/// 2026-05-18, where dropping `frames_after_eos` below the upstream default -/// of 3 clipped the leading audio of multi-clause sentences. +/// this threshold we leave sherpa-onnx's own defaults in place because +/// dropping `frames_after_eos` below the upstream default of 3 can clip the +/// leading audio of multi-clause sentences. const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; /// Number of leading spaces prepended to short prompts. The upstream Python @@ -132,21 +125,14 @@ const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; /// This is upstream's *only* mitigation for the FlowLM cold-start smear on /// short utterances (kyutai-labs/pocket-tts #91, #70): the autoregressive /// generation has a 2–3 step "settle" period where the first phoneme can be -/// smeared. A previous revision added a sacrificial `". . "` prefix plus an -/// amplitude-threshold trim to strip the rendered prefix from the output — -/// but the trim's absolute threshold (0.02 against raw peaks of ~0.076) sat -/// in soft-onset territory and could eat real word starts, and its tuning -/// was calibrated against `silence_scale = 0.0` audio. Deleted in favour of -/// upstream parity: accept the occasional smeared first syllable rather -/// than risk trimming real speech. +/// smeared. The pad must remain whitespace-only: synthesized sacrificial text +/// requires amplitude-threshold trimming, which can eat soft word starts. const SHORT_PROMPT_PAD_SPACES: usize = 8; /// sherpa-onnx's documented `frames_after_eos` default. We deliberately do -/// *not* override this knob — the previous attempt to bump it for short -/// inputs and lower it for long inputs lowered it below the upstream default -/// of 3, which clipped the leading audio of multi-clause sentences (the -/// "first 'yep' is static" regression). The constant exists only for the -/// regression test below. Source: `offline-tts-pocket-impl.h:Generate`. +/// *not* override this knob because values below the upstream default of 3 can +/// clip the leading audio of multi-clause sentences. The constant exists only +/// for the invariant test below. Source: `offline-tts-pocket-impl.h:Generate`. #[cfg(test)] const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; @@ -367,12 +353,9 @@ pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { /// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. /// -/// Centralised so the regression test below can assert that we **never** -/// emit a `frames_after_eos` override — the previous attempt to override -/// that knob (setting it to 1 for ≥5-word inputs) clipped the leading -/// audio of multi-clause sentences (the "first 'yep' is static" bug on -/// 2026-05-18). The upstream sherpa-onnx default of 3 is what we want, and -/// the right way to keep it is to not set it at all. +/// Centralised so the invariant test below can assert that we **never** emit a +/// `frames_after_eos` override. Leaving the key unset preserves sherpa-onnx's +/// upstream default of 3. fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { prepared.max_frames.map(|mf| { let mut h: HashMap = HashMap::with_capacity(1); @@ -567,17 +550,9 @@ mod tests { // ── build_generation_extra ─────────────────────────────────────────────── // - // These tests pin down a behaviour we've now regressed twice on: - // 1) Not padding/punctuating short inputs → 40 s of "monster breathing" - // (pre-773a2a1). - // 2) Setting `frames_after_eos = 1` on long inputs → clipped leading - // audio of multi-clause sentences, e.g. "Yep, I can hear you. …" - // came out as a static burst (the 773a2a1 regression Tyler hit on - // 2026-05-18 ~14:30 UTC). - // - // The contract we enforce going forward: we **only** override - // `max_frames`, and only for ≤4-word inputs. Every other knob is left - // at sherpa-onnx's documented default (notably `frames_after_eos = 3`). + // Short prompts override only `max_frames`; long prompts emit no extras. + // Every other knob remains at sherpa-onnx's documented default, notably + // `frames_after_eos = 3`. #[test] fn build_extra_short_prompt_sets_only_max_frames() { @@ -597,9 +572,7 @@ mod tests { #[test] fn build_extra_long_prompt_is_none() { - // ≥5 words: no extras at all. This is the key fix for the "first - // 'yep' in 'Yep, I can hear you. …' is static" regression — we - // were previously forcing `frames_after_eos = 1` on this path. + // ≥5 words: no extras, so the upstream LM defaults remain authoritative. let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); assert_eq!( build_generation_extra(&prepared), From 44ce8b0cddae0a0d8c68cce83f02d7292e6cda1f Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 28 Jul 2026 04:20:49 -0400 Subject: [PATCH 3/8] feat(voice): expose mobile linking and cancellation Signed-off-by: John Tennant --- crates/buzz-voice/Cargo.toml | 7 ++++++- crates/buzz-voice/src/pocket.rs | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/crates/buzz-voice/Cargo.toml b/crates/buzz-voice/Cargo.toml index f577e3d1a6..b568bb4a05 100644 --- a/crates/buzz-voice/Cargo.toml +++ b/crates/buzz-voice/Cargo.toml @@ -7,6 +7,11 @@ license.workspace = true repository.workspace = true description = "Reusable local voice primitives for Buzz" +[features] +default = ["static"] +static = ["sherpa-onnx/static"] +shared = ["sherpa-onnx/shared"] + [dependencies] serde_json = { workspace = true } -sherpa-onnx = "1.12" +sherpa-onnx = { version = "1.12", default-features = false } diff --git a/crates/buzz-voice/src/pocket.rs b/crates/buzz-voice/src/pocket.rs index e7daa3ed96..277fecd3d6 100644 --- a/crates/buzz-voice/src/pocket.rs +++ b/crates/buzz-voice/src/pocket.rs @@ -372,12 +372,31 @@ impl PocketTts { /// directly and is a one-step consistency model. Returns an empty buffer /// for whitespace-only input. pub fn synth_chunk( + &self, + text: &str, + lang: &str, + style: &VoiceStyle, + steps: usize, + ) -> Result, String> { + self.synth_chunk_with_callback(text, lang, style, steps, None:: bool>) + } + + /// Synthesise `text`, allowing the caller to stop generation early. + /// + /// The callback receives the samples generated so far and a progress + /// value in `[0, 1]`. Return `true` to continue or `false` to cancel. + /// Clients that do not need cancellation should use [`Self::synth_chunk`]. + pub fn synth_chunk_with_callback( &self, text: &str, _lang: &str, style: &VoiceStyle, _steps: usize, - ) -> Result, String> { + callback: Option, + ) -> Result, String> + where + F: FnMut(&[f32], f32) -> bool + 'static, + { // Mirror upstream pocket-tts prompt prep — without this short or // unpunctuated inputs can cause the LM's EOS logit to never trip, // producing up to 40 s of "monster breathing" garbage on the first @@ -412,7 +431,7 @@ impl PocketTts { // `generate_with_config` generic parameter. let audio = self .inner - .generate_with_config(&prepared.text, &cfg, None:: bool>) + .generate_with_config(&prepared.text, &cfg, callback) .ok_or_else(|| { format!( "Pocket TTS synthesis failed for text ({} chars)", From 55eae44dc33dfefeb1794e0b44e31fe7afe014db Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 28 Jul 2026 04:40:48 -0400 Subject: [PATCH 4/8] fix(voice): link official sherpa mobile libraries Signed-off-by: John Tennant --- Cargo.lock | 2 - Cargo.toml | 2 + crates/sherpa-onnx-sys/Cargo.toml | 63 ++ crates/sherpa-onnx-sys/Cargo.toml.orig | 34 + crates/sherpa-onnx-sys/LICENSE | 202 ++++++ crates/sherpa-onnx-sys/README.md | 671 ++++++++++++++++++ crates/sherpa-onnx-sys/build.rs | 475 +++++++++++++ crates/sherpa-onnx-sys/src/audio_tagging.rs | 60 ++ crates/sherpa-onnx-sys/src/kws.rs | 88 +++ crates/sherpa-onnx-sys/src/lib.rs | 42 ++ crates/sherpa-onnx-sys/src/offline_asr.rs | 281 ++++++++ .../src/offline_punctuation.rs | 40 ++ .../src/offline_speaker_diarization.rs | 98 +++ crates/sherpa-onnx-sys/src/online_asr.rs | 202 ++++++ .../sherpa-onnx-sys/src/online_punctuation.rs | 41 ++ crates/sherpa-onnx-sys/src/resampler.rs | 42 ++ .../sherpa-onnx-sys/src/speaker_embedding.rs | 131 ++++ crates/sherpa-onnx-sys/src/speech_denoiser.rs | 88 +++ .../src/spoken_language_identification.rs | 54 ++ crates/sherpa-onnx-sys/src/tts.rs | 172 +++++ crates/sherpa-onnx-sys/src/vad.rs | 85 +++ crates/sherpa-onnx-sys/src/wave.rs | 30 + 22 files changed, 2901 insertions(+), 2 deletions(-) create mode 100644 crates/sherpa-onnx-sys/Cargo.toml create mode 100644 crates/sherpa-onnx-sys/Cargo.toml.orig create mode 100644 crates/sherpa-onnx-sys/LICENSE create mode 100644 crates/sherpa-onnx-sys/README.md create mode 100644 crates/sherpa-onnx-sys/build.rs create mode 100644 crates/sherpa-onnx-sys/src/audio_tagging.rs create mode 100644 crates/sherpa-onnx-sys/src/kws.rs create mode 100644 crates/sherpa-onnx-sys/src/lib.rs create mode 100644 crates/sherpa-onnx-sys/src/offline_asr.rs create mode 100644 crates/sherpa-onnx-sys/src/offline_punctuation.rs create mode 100644 crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs create mode 100644 crates/sherpa-onnx-sys/src/online_asr.rs create mode 100644 crates/sherpa-onnx-sys/src/online_punctuation.rs create mode 100644 crates/sherpa-onnx-sys/src/resampler.rs create mode 100644 crates/sherpa-onnx-sys/src/speaker_embedding.rs create mode 100644 crates/sherpa-onnx-sys/src/speech_denoiser.rs create mode 100644 crates/sherpa-onnx-sys/src/spoken_language_identification.rs create mode 100644 crates/sherpa-onnx-sys/src/tts.rs create mode 100644 crates/sherpa-onnx-sys/src/vad.rs create mode 100644 crates/sherpa-onnx-sys/src/wave.rs diff --git a/Cargo.lock b/Cargo.lock index 5a57e203d7..84c849edf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8091,8 +8091,6 @@ dependencies = [ [[package]] name = "sherpa-onnx-sys" version = "1.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc951af03dc0653c0622158ca8a585a6f2bc43b7b06048cf0e5b5020005c227" dependencies = [ "bzip2", "tar", diff --git a/Cargo.toml b/Cargo.toml index 3268cfaf8d..4761ef4f95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ members = [ "crates/buzz-relay-mesh", "crates/buzz-dev-mcp", "crates/buzz-voice", + "crates/sherpa-onnx-sys", "examples/countdown-bot", ] exclude = ["desktop/src-tauri"] @@ -171,3 +172,4 @@ strip = true # allowlist for the auth token). Revert to crates.io once #449 lands upstream. [patch.crates-io] aws-creds = { git = "https://github.com/tlongwell-block/rust-s3", rev = "c9fce3620dd434c1f810101d672cf384268dbb0f" } +sherpa-onnx-sys = { path = "crates/sherpa-onnx-sys" } diff --git a/crates/sherpa-onnx-sys/Cargo.toml b/crates/sherpa-onnx-sys/Cargo.toml new file mode 100644 index 0000000000..317d2c5b26 --- /dev/null +++ b/crates/sherpa-onnx-sys/Cargo.toml @@ -0,0 +1,63 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "sherpa-onnx-sys" +version = "1.13.4" +build = "build.rs" +links = "sherpa-onnx" +include = [ + "src/**", + "build.rs", + "Cargo.toml", + "README.md", + "LICENSE*", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Raw FFI bindings to the sherpa-onnx C API" +readme = "README.md" +keywords = [ + "ffi", + "speech", + "sherpa-onnx", + "bindings", +] +categories = ["external-ffi-bindings"] +license = "Apache-2.0" +repository = "https://github.com/k2-fsa/sherpa-onnx" + +[package.metadata.docs.rs] +default-features = false +features = [] + +[features] +default = ["static"] +shared = [] +static = [] + +[lib] +name = "sherpa_onnx_sys" +path = "src/lib.rs" + +[build-dependencies.bzip2] +version = "0.4" + +[build-dependencies.tar] +version = "0.4" + +[build-dependencies.ureq] +version = "2.12" +features = ["proxy-from-env"] diff --git a/crates/sherpa-onnx-sys/Cargo.toml.orig b/crates/sherpa-onnx-sys/Cargo.toml.orig new file mode 100644 index 0000000000..5744d49363 --- /dev/null +++ b/crates/sherpa-onnx-sys/Cargo.toml.orig @@ -0,0 +1,34 @@ +[package] +name = "sherpa-onnx-sys" +version = "1.13.4" +edition = "2021" +description = "Raw FFI bindings to the sherpa-onnx C API" +license = "Apache-2.0" +repository = "https://github.com/k2-fsa/sherpa-onnx" +readme = "README.md" +links = "sherpa-onnx" + +keywords = ["ffi", "speech", "sherpa-onnx", "bindings"] +categories = ["external-ffi-bindings"] + +include = [ + "src/**", + "build.rs", + "Cargo.toml", + "README.md", + "LICENSE*", +] + +[features] +default = ["static"] +static = [] +shared = [] + +[build-dependencies] +bzip2 = "0.4" +tar = "0.4" +ureq = { version = "2.12", features = ["proxy-from-env"] } + +[package.metadata.docs.rs] +default-features = false +features = [] diff --git a/crates/sherpa-onnx-sys/LICENSE b/crates/sherpa-onnx-sys/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/crates/sherpa-onnx-sys/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/sherpa-onnx-sys/README.md b/crates/sherpa-onnx-sys/README.md new file mode 100644 index 0000000000..06db754497 --- /dev/null +++ b/crates/sherpa-onnx-sys/README.md @@ -0,0 +1,671 @@ +
+ +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/k2-fsa/sherpa-onnx) + +
+ +Buzz carries this source-identical crates.io package so its build script can +link the official merged iOS archives and Android shared libraries through +`SHERPA_ONNX_LIB_DIR`. Desktop archive selection remains unchanged. + + ### Supported functions + +|Speech recognition| [Speech synthesis][tts-url] | [Source separation][ss-url] | +|------------------|------------------|-------------------| +| ✔️ | ✔️ | ✔️ | + +|Speaker identification| [Speaker diarization][sd-url] | Speaker verification | +|----------------------|-------------------- |------------------------| +| ✔️ | ✔️ | ✔️ | + +| [Spoken Language identification][slid-url] | [Audio tagging][at-url] | [Voice activity detection][vad-url] | +|--------------------------------|---------------|--------------------------| +| ✔️ | ✔️ | ✔️ | + +| [Keyword spotting][kws-url] | [Add punctuation][punct-url] | [Speech enhancement][se-url] | +|------------------|-----------------|--------------------| +| ✔️ | ✔️ | ✔️ | + + +### Supported platforms + +|Architecture| Android | iOS | Windows | macOS | linux | HarmonyOS | +|------------|---------|---------|------------|-------|-------|-----------| +| x64 | ✔️ | | ✔️ | ✔️ | ✔️ | ✔️ | +| x86 | ✔️ | | ✔️ | | | | +| arm64 | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | +| arm32 | ✔️ | | | | ✔️ | ✔️ | +| riscv64 | | | | | ✔️ | | + +### Supported programming languages + +| 1. C++ | 2. C | 3. Python | 4. JavaScript | +|--------|-------|-----------|---------------| +| ✔️ | ✔️ | ✔️ | ✔️ | + +|5. Java | 6. C# | 7. Kotlin | 8. Swift | +|--------|-------|-----------|----------| +| ✔️ | ✔️ | ✔️ | ✔️ | + +| 9. Go | 10. Dart | 11. Rust | 12. Pascal | +|-------|----------|----------|------------| +| ✔️ | ✔️ | ✔️ | ✔️ | + + +It also supports WebAssembly. + +### Supported NPUs + +| [1. Rockchip NPU (RKNN)][rknpu-doc] | [2. Qualcomm NPU (QNN)][qnn-doc] | [3. Ascend NPU][ascend-doc] | +|-------------------------------------|-----------------------------------|-----------------------------| +| ✔️ | ✔️ | ✔️ | + +| [4. Axera NPU][axera-npu] | +|---------------------------| +| ✔️ | + +[Join our discord](https://discord.gg/fJdxzg2VbG) + + +## Introduction + +This repository supports running the following functions **locally** + + - Speech-to-text (i.e., ASR); both streaming and non-streaming are supported + - Text-to-speech (i.e., TTS) + - Speaker diarization + - Speaker identification + - Speaker verification + - Spoken language identification + - Audio tagging + - VAD (e.g., [silero-vad][silero-vad]) + - Speech enhancement (e.g., [gtcrn][gtcrn], [DPDFNet](https://github.com/ceva-ip/DPDFNet)) + - Keyword spotting + - Source separation (e.g., [spleeter][spleeter], [UVR][UVR]) + +on the following platforms and operating systems: + + - x86, ``x86_64``, 32-bit ARM, 64-bit ARM (arm64, aarch64), RISC-V (riscv64), **RK NPU**, **Ascend NPU** + - Linux, macOS, Windows, openKylin + - Android, WearOS + - iOS + - HarmonyOS + - NodeJS + - WebAssembly + - [NVIDIA Jetson Orin NX][NVIDIA Jetson Orin NX] (Support running on both CPU and GPU) + - [NVIDIA Jetson Nano B01][NVIDIA Jetson Nano B01] (Support running on both CPU and GPU) + - [Raspberry Pi][Raspberry Pi] + - [RV1126][RV1126] + - [LicheePi4A][LicheePi4A] + - [VisionFive 2][VisionFive 2] + - [旭日X3派][旭日X3派] + - [爱芯派][爱芯派] + - [RK3588][RK3588] + - [SpacemiT-K1][SpacemiT-K1] + - [SpacemiT-K3][SpacemiT-K3] + - etc + +with the following APIs + + - C++, C, Python, Go, ``C#`` + - Java, Kotlin, JavaScript + - Swift, Rust + - Dart, Object Pascal + +### Links for Huggingface Spaces + +
+You can visit the following Huggingface spaces to try sherpa-onnx without +installing anything. All you need is a browser. + +| Description | URL | 中国镜像 | +|-------------------------------------------------------|-----------------------------------------|----------------------------------------| +| Speaker diarization | [Click me][hf-space-speaker-diarization]| [镜像][hf-space-speaker-diarization-cn]| +| Speech recognition | [Click me][hf-space-asr] | [镜像][hf-space-asr-cn] | +| Speech recognition with [Whisper][Whisper] | [Click me][hf-space-asr-whisper] | [镜像][hf-space-asr-whisper-cn] | +| Speech synthesis | [Click me][hf-space-tts] | [镜像][hf-space-tts-cn] | +| Generate subtitles | [Click me][hf-space-subtitle] | [镜像][hf-space-subtitle-cn] | +| Audio tagging | [Click me][hf-space-audio-tagging] | [镜像][hf-space-audio-tagging-cn] | +| Source separation | [Click me][hf-space-source-separation] | [镜像][hf-space-source-separation-cn] | +| Spoken language identification with [Whisper][Whisper]| [Click me][hf-space-slid-whisper] | [镜像][hf-space-slid-whisper-cn] | + +We also have spaces built using WebAssembly. They are listed below: + +| Description | Huggingface space| ModelScope space| +|------------------------------------------------------------------------------------------|------------------|-----------------| +|Voice activity detection with [silero-vad][silero-vad] | [Click me][wasm-hf-vad]|[地址][wasm-ms-vad]| +|Real-time speech recognition (Chinese + English) with Zipformer | [Click me][wasm-hf-streaming-asr-zh-en-zipformer]|[地址][wasm-hf-streaming-asr-zh-en-zipformer]| +|Real-time speech recognition (Chinese + English) with Paraformer |[Click me][wasm-hf-streaming-asr-zh-en-paraformer]| [地址][wasm-ms-streaming-asr-zh-en-paraformer]| +|Real-time speech recognition (Chinese + English + Cantonese) with [Paraformer-large][Paraformer-large]|[Click me][wasm-hf-streaming-asr-zh-en-yue-paraformer]| [地址][wasm-ms-streaming-asr-zh-en-yue-paraformer]| +|Real-time speech recognition (English) |[Click me][wasm-hf-streaming-asr-en-zipformer] |[地址][wasm-ms-streaming-asr-en-zipformer]| +|VAD + speech recognition (Chinese) with [Zipformer CTC](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/icefall/zipformer.html#sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03-chinese)|[Click me][wasm-hf-vad-asr-zh-zipformer-ctc-07-03]| [地址][wasm-ms-vad-asr-zh-zipformer-ctc-07-03]| +|VAD + speech recognition (Chinese + English + Korean + Japanese + Cantonese) with [SenseVoice][SenseVoice]|[Click me][wasm-hf-vad-asr-zh-en-ko-ja-yue-sense-voice]| [地址][wasm-ms-vad-asr-zh-en-ko-ja-yue-sense-voice]| +|VAD + speech recognition (English) with [Whisper][Whisper] tiny.en|[Click me][wasm-hf-vad-asr-en-whisper-tiny-en]| [地址][wasm-ms-vad-asr-en-whisper-tiny-en]| +|VAD + speech recognition (English) with [Moonshine tiny][Moonshine tiny]|[Click me][wasm-hf-vad-asr-en-moonshine-tiny-en]| [地址][wasm-ms-vad-asr-en-moonshine-tiny-en]| +|VAD + speech recognition (English) with Zipformer trained with [GigaSpeech][GigaSpeech] |[Click me][wasm-hf-vad-asr-en-zipformer-gigaspeech]| [地址][wasm-ms-vad-asr-en-zipformer-gigaspeech]| +|VAD + speech recognition (Chinese) with Zipformer trained with [WenetSpeech][WenetSpeech] |[Click me][wasm-hf-vad-asr-zh-zipformer-wenetspeech]| [地址][wasm-ms-vad-asr-zh-zipformer-wenetspeech]| +|VAD + speech recognition (Japanese) with Zipformer trained with [ReazonSpeech][ReazonSpeech]|[Click me][wasm-hf-vad-asr-ja-zipformer-reazonspeech]| [地址][wasm-ms-vad-asr-ja-zipformer-reazonspeech]| +|VAD + speech recognition (Thai) with Zipformer trained with [GigaSpeech2][GigaSpeech2] |[Click me][wasm-hf-vad-asr-th-zipformer-gigaspeech2]| [地址][wasm-ms-vad-asr-th-zipformer-gigaspeech2]| +|VAD + speech recognition (Chinese 多种方言) with a [TeleSpeech-ASR][TeleSpeech-ASR] CTC model|[Click me][wasm-hf-vad-asr-zh-telespeech]| [地址][wasm-ms-vad-asr-zh-telespeech]| +|VAD + speech recognition (English + Chinese, 及多种中文方言) with Paraformer-large |[Click me][wasm-hf-vad-asr-zh-en-paraformer-large]| [地址][wasm-ms-vad-asr-zh-en-paraformer-large]| +|VAD + speech recognition (English + Chinese, 及多种中文方言) with Paraformer-small |[Click me][wasm-hf-vad-asr-zh-en-paraformer-small]| [地址][wasm-ms-vad-asr-zh-en-paraformer-small]| +|VAD + speech recognition (多语种及多种中文方言) with [Dolphin][Dolphin]-base |[Click me][wasm-hf-vad-asr-multi-lang-dolphin-base]| [地址][wasm-ms-vad-asr-multi-lang-dolphin-base]| +|Speech synthesis (Piper, English) |[Click me][wasm-hf-tts-piper-en]| [地址][wasm-ms-tts-piper-en]| +|Speech synthesis (Piper, German) |[Click me][wasm-hf-tts-piper-de]| [地址][wasm-ms-tts-piper-de]| +|Speech synthesis (Matcha, Chinese) |[Click me][wasm-hf-tts-matcha-zh]| [地址][wasm-ms-tts-matcha-zh]| +|Speech synthesis (Matcha, English) |[Click me][wasm-hf-tts-matcha-en]| [地址][wasm-ms-tts-matcha-en]| +|Speech synthesis (Matcha, Chinese+English) |[Click me][wasm-hf-tts-matcha-zh-en]| [地址][wasm-ms-tts-matcha-zh-en]| +|Speaker diarization |[Click me][wasm-hf-speaker-diarization]|[地址][wasm-ms-speaker-diarization]| +|Voice cloning with ZipVoice (Chinese+English) |[Click me][wasm-hf-voice-cloning-zipvoice]|[地址][wasm-ms-voice-cloning-zipvoice]| +|Voice cloning with Pocket TTS (English) |[Click me][wasm-hf-voice-cloning-pocket]|[地址][wasm-ms-voice-cloning-pocket]| + +
+ +### Links for pre-built Android APKs + +
+ +You can find pre-built Android APKs for this repository in the following table + +| Description | URL | 中国用户 | +|----------------------------------------|------------------------------------|-----------------------------------| +| Speaker diarization | [Address][apk-speaker-diarization] | [点此][apk-speaker-diarization-cn]| +| Streaming speech recognition | [Address][apk-streaming-asr] | [点此][apk-streaming-asr-cn] | +| Simulated-streaming speech recognition | [Address][apk-simula-streaming-asr]| [点此][apk-simula-streaming-asr-cn]| +| Text-to-speech | [Address][apk-tts] | [点此][apk-tts-cn] | +| Voice activity detection (VAD) | [Address][apk-vad] | [点此][apk-vad-cn] | +| VAD + non-streaming speech recognition | [Address][apk-vad-asr] | [点此][apk-vad-asr-cn] | +| Two-pass speech recognition | [Address][apk-2pass] | [点此][apk-2pass-cn] | +| Audio tagging | [Address][apk-at] | [点此][apk-at-cn] | +| Audio tagging (WearOS) | [Address][apk-at-wearos] | [点此][apk-at-wearos-cn] | +| Speaker identification | [Address][apk-sid] | [点此][apk-sid-cn] | +| Spoken language identification | [Address][apk-slid] | [点此][apk-slid-cn] | +| Keyword spotting | [Address][apk-kws] | [点此][apk-kws-cn] | + +
+ +### Links for pre-built Flutter APPs + +
+ +#### Real-time speech recognition + +| Description | URL | 中国用户 | +|--------------------------------|-------------------------------------|-------------------------------------| +| Streaming speech recognition | [Address][apk-flutter-streaming-asr]| [点此][apk-flutter-streaming-asr-cn]| + +#### Text-to-speech + +| Description | URL | 中国用户 | +|------------------------------------------|------------------------------------|------------------------------------| +| Android (arm64-v8a, armeabi-v7a, x86_64) | [Address][flutter-tts-android] | [点此][flutter-tts-android-cn] | +| Linux (x64) | [Address][flutter-tts-linux] | [点此][flutter-tts-linux-cn] | +| macOS (x64) | [Address][flutter-tts-macos-x64] | [点此][flutter-tts-macos-x64-cn] | +| macOS (arm64) | [Address][flutter-tts-macos-arm64] | [点此][flutter-tts-macos-arm64-cn] | +| Windows (x64) | [Address][flutter-tts-win-x64] | [点此][flutter-tts-win-x64-cn] | + +> Note: You need to build from source for iOS. + +
+ +### Links for pre-built Lazarus APPs + +
+ +#### Generating subtitles + +| Description | URL | 中国用户 | +|--------------------------------|----------------------------|----------------------------| +| Generate subtitles (生成字幕) | [Address][lazarus-subtitle]| [点此][lazarus-subtitle-cn]| + +
+ +### Links for pre-trained models + +
+ +| Description | URL | +|---------------------------------------------|---------------------------------------------------------------------------------------| +| Speech recognition (speech to text, ASR) | [Address][asr-models] | +| Text-to-speech (TTS) | [Address][tts-models] | +| VAD | [Address][vad-models] | +| Keyword spotting | [Address][kws-models] | +| Audio tagging | [Address][at-models] | +| Speaker identification (Speaker ID) | [Address][sid-models] | +| Spoken language identification (Language ID)| See multi-lingual [Whisper][Whisper] ASR models from [Speech recognition][asr-models]| +| Punctuation | [Address][punct-models] | +| Speaker segmentation | [Address][speaker-segmentation-models] | +| Speech enhancement | [Address][speech-enhancement-models] | +| Source separation | [Address][source-separation-models] | + +
+ +#### Some pre-trained ASR models (Streaming) + +
+ +Please see + + - + - + - + +for more models. The following table lists only **SOME** of them. + + +|Name | Supported Languages| Description| +|-----|-----|----| +|[sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20][sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20]| Chinese, English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english)| +|[sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16][sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16]| Chinese, English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16-bilingual-chinese-english)| +|[sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23][sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23]|Chinese| Suitable for Cortex A7 CPU. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-zh-14m-2023-02-23)| +|[sherpa-onnx-streaming-zipformer-en-20M-2023-02-17][sherpa-onnx-streaming-zipformer-en-20M-2023-02-17]|English|Suitable for Cortex A7 CPU. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-en-20m-2023-02-17)| +|[sherpa-onnx-streaming-zipformer-korean-2024-06-16][sherpa-onnx-streaming-zipformer-korean-2024-06-16]|Korean| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-korean-2024-06-16-korean)| +|[sherpa-onnx-streaming-zipformer-fr-2023-04-14][sherpa-onnx-streaming-zipformer-fr-2023-04-14]|French| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#shaojieli-sherpa-onnx-streaming-zipformer-fr-2023-04-14-french)| + +
+ + +#### Some pre-trained ASR models (Non-Streaming) + +
+ +Please see + + - + - + - + - + - + +for more models. The following table lists only **SOME** of them. + +|Name | Supported Languages| Description| +|-----|-----|----| +|[sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/nemo-transducer-models.html#sherpa-onnx-nemo-parakeet-tdt-0-6b-v2-int8-english)| English | It is converted from | +|[Whisper tiny.en](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2)|English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html)| +|[Moonshine tiny][Moonshine tiny]|English|See [also](https://github.com/usefulsensors/moonshine)| +|[sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/icefall/zipformer.html#sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03-chinese)|Chinese| A Zipformer CTC model| +|[sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17][sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]|Chinese, Cantonese, English, Korean, Japanese| 支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/sense-voice/index.html)| +|[sherpa-onnx-paraformer-zh-2024-03-09][sherpa-onnx-paraformer-zh-2024-03-09]|Chinese, English| 也支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2024-03-09-chinese-english)| +|[sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01][sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01]|Japanese|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01-japanese)| +|[sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24][sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24]|Russian|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/nemo-transducer-models.html#sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24-russian)| +|[sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24][sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24]|Russian| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/nemo/russian.html#sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24)| +|[sherpa-onnx-zipformer-ru-2024-09-18][sherpa-onnx-zipformer-ru-2024-09-18]|Russian|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-ru-2024-09-18-russian)| +|[sherpa-onnx-zipformer-korean-2024-06-24][sherpa-onnx-zipformer-korean-2024-06-24]|Korean|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-korean-2024-06-24-korean)| +|[sherpa-onnx-zipformer-thai-2024-06-20][sherpa-onnx-zipformer-thai-2024-06-20]|Thai| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-thai-2024-06-20-thai)| +|[sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04][sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04]|Chinese| 支持多种方言. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/telespeech/models.html#sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04)| + +
+ +### Useful links + +- Documentation: https://k2-fsa.github.io/sherpa/onnx/ +- Bilibili 演示视频: https://search.bilibili.com/all?keyword=%E6%96%B0%E4%B8%80%E4%BB%A3Kaldi + +### How to reach us + +Please see +https://k2-fsa.github.io/sherpa/social-groups.html +for 新一代 Kaldi **微信交流群** and **QQ 交流群**. + +## Projects using sherpa-onnx + +### [Sherpa Voice / @siteed/sherpa-onnx.rn](https://github.com/deeeed/audiolab) + +> React Native wrapper and demo app for validating sherpa-onnx on iOS, +> Android, and Web, including ASR, TTS, VAD, KWS, speaker ID, diarization, +> language ID, punctuation, audio tagging, and speech enhancement. + +- [NPM package](https://www.npmjs.com/package/@siteed/sherpa-onnx.rn) +- [Live demo](https://deeeed.github.io/audiolab/sherpa-voice/) + +### [Speed of Sound](https://github.com/zugaldia/speedofsound) + +> A voice-typing application for the Linux desktop (GTK4/Adwaita). +> It captures microphone audio, transcribes it offline using Sherpa ONNX ASR models, +> optionally polishes the text with an LLM, and types the result into the active window +> via XDG Remote Desktop Portal keyboard simulation. + +### [VoxSherpa TTS](https://github.com/CodeBySonu95/VoxSherpa-TTS) + +> VoxSherpa TTS is a 100% offline Android Text-to-Speech app powered by Sherpa-ONNX. +> It supports Kokoro-82M, Piper, and VITS engines with multilingual support including +> Hindi, English, British English, Japanese, Chinese and 50+ more languages. + +- [Download APK (All Versions)](https://github.com/CodeBySonu95/VoxSherpa-TTS/releases) +- Android 11+ · 100% offline · No telemetry + +
+ +| Generate | Models | Library | Settings | +|:---:|:---:|:---:|:---:| +| | | | | + +
+ +--- +### [BreezeApp](https://github.com/mtkresearch/BreezeApp) from [MediaTek Research](https://github.com/mtkresearch) + +> BreezeAPP is a mobile AI application developed for both Android and iOS platforms. +> Users can download it directly from the App Store and enjoy a variety of features +> offline, including speech-to-text, text-to-speech, text-based chatbot interactions, +> and image question-answering + + - [Download APK for BreezeAPP](https://huggingface.co/MediaTek-Research/BreezeApp/resolve/main/BreezeApp.apk) + - [APK 中国镜像](https://hf-mirror.com/MediaTek-Research/BreezeApp/blob/main/BreezeApp.apk) + +| 1 | 2 | 3 | +|---|---|---| +|![](https://github.com/user-attachments/assets/1cdbc057-b893-4de6-9e9c-f1d7dfd1d992)|![](https://github.com/user-attachments/assets/d77cd98e-b057-442f-860d-d5befd5c769b)|![](https://github.com/user-attachments/assets/57e546bf-3d39-45b9-b392-b48ca4fb3c58)| + +### [Open-LLM-VTuber](https://github.com/t41372/Open-LLM-VTuber) + +Talk to any LLM with hands-free voice interaction, voice interruption, and Live2D taking +face running locally across platforms + +See also + +### [voiceapi](https://github.com/ruzhila/voiceapi) + +
+ Streaming ASR and TTS based on FastAPI + + +It shows how to use the ASR and TTS Python APIs with FastAPI. +
+ +### [腾讯会议摸鱼工具 TMSpeech](https://github.com/jxlpzqc/TMSpeech) + +Uses streaming ASR in C# with graphical user interface. + +Video demo in Chinese: [【开源】Windows实时字幕软件（网课/开会必备）](https://www.bilibili.com/video/BV1rX4y1p7Nx) + +### [lol互动助手](https://github.com/l1veIn/lol-wom-electron) + +It uses the JavaScript API of sherpa-onnx along with [Electron](https://electronjs.org/) + +Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率的英雄联盟工具！英雄联盟的最后一块拼图！和游戏中的每个人无障碍沟通！](https://www.bilibili.com/video/BV142tje9E74) + +### [Sherpa-ONNX 语音识别服务器](https://github.com/hfyydd/sherpa-onnx-server) + +A server based on nodejs providing Restful API for speech recognition. + +### [QSmartAssistant](https://github.com/xinhecuican/QSmartAssistant) + +一个模块化，全过程可离线，低占用率的对话机器人/智能音箱 + +It uses QT. Both [ASR](https://github.com/xinhecuican/QSmartAssistant/blob/master/doc/%E5%AE%89%E8%A3%85.md#asr) +and [TTS](https://github.com/xinhecuican/QSmartAssistant/blob/master/doc/%E5%AE%89%E8%A3%85.md#tts) +are used. + +### [Flutter-EasySpeechRecognition](https://github.com/Jason-chen-coder/Flutter-EasySpeechRecognition) + +It extends [./flutter-examples/streaming_asr](./flutter-examples/streaming_asr) by +downloading models inside the app to reduce the size of the app. + +Note: [[Team B] Sherpa AI backend](https://github.com/umgc/spring2025/pull/82) also uses +sherpa-onnx in a Flutter APP. + +### [sherpa-onnx-unity](https://github.com/xue-fei/sherpa-onnx-unity) + +sherpa-onnx in Unity. See also [#1695](https://github.com/k2-fsa/sherpa-onnx/issues/1695), +[#1892](https://github.com/k2-fsa/sherpa-onnx/issues/1892), and [#1859](https://github.com/k2-fsa/sherpa-onnx/issues/1859) + +### [xiaozhi-esp32-server](https://github.com/xinnan-tech/xiaozhi-esp32-server) + +本项目为xiaozhi-esp32提供后端服务，帮助您快速搭建ESP32设备控制服务器 +Backend service for xiaozhi-esp32, helps you quickly build an ESP32 device control server. + +See also + + - [ASR新增轻量级sherpa-onnx-asr](https://github.com/xinnan-tech/xiaozhi-esp32-server/issues/315) + - [feat: ASR增加sherpa-onnx模型](https://github.com/xinnan-tech/xiaozhi-esp32-server/pull/379) + +### [KaithemAutomation](https://github.com/EternityForest/KaithemAutomation) + +Pure Python, GUI-focused home automation/consumer grade SCADA. + +It uses TTS from sherpa-onnx. See also [✨ Speak command that uses the new globally configured TTS model.](https://github.com/EternityForest/KaithemAutomation/commit/8e64d2b138725e426532f7d66bb69dd0b4f53693) + +### [Open-XiaoAI KWS](https://github.com/idootop/open-xiaoai-kws) + +Enable custom wake word for XiaoAi Speakers. 让小爱音箱支持自定义唤醒词。 + +Video demo in Chinese: [小爱同学启动～˶╹ꇴ╹˶！](https://www.bilibili.com/video/BV1YfVUz5EMj) + +### [C++ WebSocket ASR Server](https://github.com/mawwalker/stt-server) + +It provides a WebSocket server based on C++ for ASR using sherpa-onnx. + +### [Go WebSocket Server](https://github.com/bbeyondllove/asr_server) + +It provides a WebSocket server based on the Go programming language for sherpa-onnx. + +### [Making robot Paimon, Ep10 "The AI Part 1"](https://www.youtube.com/watch?v=KxPKkwxGWZs) + +It is a [YouTube video](https://www.youtube.com/watch?v=KxPKkwxGWZs), +showing how the author tried to use AI so he can have a conversation with Paimon. + +It uses sherpa-onnx for speech-to-text and text-to-speech. +|1| +|---| +|![](https://github.com/user-attachments/assets/f6eea2d5-1807-42cb-9160-be8da2971e1f)| + +### [TtsReader - Desktop application](https://github.com/ys-pro-duction/TtsReader) + +A desktop text-to-speech application built using Kotlin Multiplatform. + +### [MentraOS](https://github.com/Mentra-Community/MentraOS) + +> Smart glasses OS, with dozens of built-in apps. Users get AI assistant, notifications, +> translation, screen mirror, captions, and more. Devs get to write 1 app that runs on +> any pair of smart glasses. + +It uses sherpa-onnx for real-time speech recognition on iOS and Android devices. +See also + +It uses Swift for iOS and Java for Android. + +### [flet_sherpa_onnx](https://github.com/SamYuan1990/flet_sherpa_onnx) + +Flet ASR/STT component based on sherpa-onnx. +Example [a chat box agent](https://github.com/SamYuan1990/i18n-agent-action) + +### [achatbot-go](https://github.com/ai-bot-pro/achatbot-go) + +a multimodal chatbot based on go with sherpa-onnx's speech lib api. + +### [fcitx5-vinput](https://github.com/xifan2333/fcitx5-vinput) + +Local offline voice input plugin for [Fcitx5](https://github.com/fcitx/fcitx5) (Linux input method framework). +It uses C++ with offline ASR for speech recognition, supporting push-to-talk, +command mode, and optional LLM post-processing. + +Video demo in Chinese: [fcitx5-vinput](https://www.bilibili.com/video/BV1a6cUzVE6F) + +### [Wake Word](https://github.com/analyticsinmotion/wake-word) + +A VS Code extension for hands-free voice-activated coding. It uses sherpa-onnx for real-time +keyword spotting (KWS) to detect custom wake phrases and trigger VS Code commands by voice. +Audio capture is handled by [decibri](https://github.com/analyticsinmotion/decibri), a +cross-platform Node.js microphone streaming library with prebuilt native binaries. + +- [VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=analytics-in-motion.wake-word) +- [Open VSX](https://open-vsx.org/extension/analytics-in-motion/wake-word) +- [decibri integration guides for sherpa-onnx](https://decibri.dev/docs/node/integrations/sherpa-onnx-stt.html) + +### [SmartSub](https://github.com/buxuku/SmartSub) + +> SmartSub is a local-first cross-platform desktop application for the complete subtitle production pipeline: audio/video transcription, subtitle translation, proofreading, and subtitle burning/muxing. +> +> It natively integrates sherpa-onnx to power three offline ASR engines — FunASR, Qwen3-ASR, and FireRedASR — delivering high-accuracy Chinese and multilingual speech recognition entirely on-device, with no file uploads required. + +[silero-vad]: https://github.com/snakers4/silero-vad +[Raspberry Pi]: https://www.raspberrypi.com/ +[RV1126]: https://www.rock-chips.com/uploads/pdf/2022.8.26/191/RV1126%20Brief%20Datasheet.pdf +[LicheePi4A]: https://sipeed.com/licheepi4a +[VisionFive 2]: https://www.starfivetech.com/en/site/boards +[旭日X3派]: https://developer.horizon.ai/api/v1/fileData/documents_pi/index.html +[爱芯派]: https://wiki.sipeed.com/hardware/zh/maixIII/ax-pi/axpi.html +[hf-space-speaker-diarization]: https://huggingface.co/spaces/k2-fsa/speaker-diarization +[hf-space-speaker-diarization-cn]: https://hf.qhduan.com/spaces/k2-fsa/speaker-diarization +[hf-space-asr]: https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition +[hf-space-asr-cn]: https://hf.qhduan.com/spaces/k2-fsa/automatic-speech-recognition +[Whisper]: https://github.com/openai/whisper +[hf-space-asr-whisper]: https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition-with-whisper +[hf-space-asr-whisper-cn]: https://hf.qhduan.com/spaces/k2-fsa/automatic-speech-recognition-with-whisper +[hf-space-tts]: https://huggingface.co/spaces/k2-fsa/text-to-speech +[hf-space-tts-cn]: https://hf.qhduan.com/spaces/k2-fsa/text-to-speech +[hf-space-subtitle]: https://huggingface.co/spaces/k2-fsa/generate-subtitles-for-videos +[hf-space-subtitle-cn]: https://hf.qhduan.com/spaces/k2-fsa/generate-subtitles-for-videos +[hf-space-audio-tagging]: https://huggingface.co/spaces/k2-fsa/audio-tagging +[hf-space-audio-tagging-cn]: https://hf.qhduan.com/spaces/k2-fsa/audio-tagging +[hf-space-source-separation]: https://huggingface.co/spaces/k2-fsa/source-separation +[hf-space-source-separation-cn]: https://hf.qhduan.com/spaces/k2-fsa/source-separation +[hf-space-slid-whisper]: https://huggingface.co/spaces/k2-fsa/spoken-language-identification +[hf-space-slid-whisper-cn]: https://hf.qhduan.com/spaces/k2-fsa/spoken-language-identification +[wasm-hf-vad]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-sherpa-onnx +[wasm-ms-vad]: https://modelscope.cn/studios/csukuangfj/web-assembly-vad-sherpa-onnx +[wasm-hf-streaming-asr-zh-en-zipformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en +[wasm-ms-streaming-asr-zh-en-zipformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en +[wasm-hf-streaming-asr-zh-en-paraformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en-paraformer +[wasm-ms-streaming-asr-zh-en-paraformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en-paraformer +[Paraformer-large]: https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary +[wasm-hf-streaming-asr-zh-en-yue-paraformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-cantonese-en-paraformer +[wasm-ms-streaming-asr-zh-en-yue-paraformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-cantonese-en-paraformer +[wasm-hf-streaming-asr-en-zipformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-en +[wasm-ms-streaming-asr-en-zipformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-en +[SenseVoice]: https://github.com/FunAudioLLM/SenseVoice +[wasm-hf-vad-asr-zh-zipformer-ctc-07-03]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc +[wasm-ms-vad-asr-zh-zipformer-ctc-07-03]: https://modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc/summary +[wasm-hf-vad-asr-zh-en-ko-ja-yue-sense-voice]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-ja-ko-cantonese-sense-voice +[wasm-ms-vad-asr-zh-en-ko-ja-yue-sense-voice]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-zh-en-jp-ko-cantonese-sense-voice +[wasm-hf-vad-asr-en-whisper-tiny-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-whisper-tiny +[wasm-ms-vad-asr-en-whisper-tiny-en]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-en-whisper-tiny +[wasm-hf-vad-asr-en-moonshine-tiny-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny +[wasm-ms-vad-asr-en-moonshine-tiny-en]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny +[wasm-hf-vad-asr-en-zipformer-gigaspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-zipformer-gigaspeech +[wasm-ms-vad-asr-en-zipformer-gigaspeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-zipformer-gigaspeech +[wasm-hf-vad-asr-zh-zipformer-wenetspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-wenetspeech +[wasm-ms-vad-asr-zh-zipformer-wenetspeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-wenetspeech +[reazonspeech]: https://research.reazon.jp/_static/reazonspeech_nlp2023.pdf +[wasm-hf-vad-asr-ja-zipformer-reazonspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-ja-zipformer +[wasm-ms-vad-asr-ja-zipformer-reazonspeech]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-ja-zipformer +[gigaspeech2]: https://github.com/speechcolab/gigaspeech2 +[wasm-hf-vad-asr-th-zipformer-gigaspeech2]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-th-zipformer +[wasm-ms-vad-asr-th-zipformer-gigaspeech2]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-th-zipformer +[telespeech-asr]: https://github.com/tele-ai/telespeech-asr +[wasm-hf-vad-asr-zh-telespeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-telespeech +[wasm-ms-vad-asr-zh-telespeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-telespeech +[wasm-hf-vad-asr-zh-en-paraformer-large]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer +[wasm-ms-vad-asr-zh-en-paraformer-large]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer +[wasm-hf-vad-asr-zh-en-paraformer-small]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer-small +[wasm-ms-vad-asr-zh-en-paraformer-small]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer-small +[dolphin]: https://github.com/dataoceanai/dolphin +[wasm-ms-vad-asr-multi-lang-dolphin-base]: https://modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-multi-lang-dophin-ctc +[wasm-hf-vad-asr-multi-lang-dolphin-base]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-multi-lang-dophin-ctc + +[wasm-hf-tts-matcha-zh-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-en-tts-matcha +[wasm-hf-tts-matcha-zh]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-tts-matcha +[wasm-ms-tts-matcha-zh-en]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-en-tts-matcha +[wasm-ms-tts-matcha-zh]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-tts-matcha +[wasm-hf-tts-matcha-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-en-tts-matcha +[wasm-ms-tts-matcha-en]: https://modelscope.cn/studios/csukuangfj/web-assembly-en-tts-matcha +[wasm-hf-tts-piper-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-en +[wasm-ms-tts-piper-en]: https://modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-en +[wasm-hf-tts-piper-de]: https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-de +[wasm-ms-tts-piper-de]: https://modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-de +[wasm-hf-speaker-diarization]: https://huggingface.co/spaces/k2-fsa/web-assembly-speaker-diarization-sherpa-onnx +[wasm-ms-speaker-diarization]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-speaker-diarization-sherpa-onnx +[wasm-hf-voice-cloning-zipvoice]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-en-tts-zipvoice +[wasm-ms-voice-cloning-zipvoice]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-en-tts-zipvoice +[wasm-hf-voice-cloning-pocket]: https://huggingface.co/spaces/k2-fsa/web-assembly-en-tts-pocket +[wasm-ms-voice-cloning-pocket]: https://modelscope.cn/studios/csukuangfj/web-assembly-en-tts-pocket +[apk-speaker-diarization]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk.html +[apk-speaker-diarization-cn]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk-cn.html +[apk-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/android/apk.html +[apk-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-cn.html +[apk-simula-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/android/apk-simulate-streaming-asr.html +[apk-simula-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-simulate-streaming-asr-cn.html +[apk-tts]: https://k2-fsa.github.io/sherpa/onnx/tts/apk-engine.html +[apk-tts-cn]: https://k2-fsa.github.io/sherpa/onnx/tts/apk-engine-cn.html +[apk-vad]: https://k2-fsa.github.io/sherpa/onnx/vad/apk.html +[apk-vad-cn]: https://k2-fsa.github.io/sherpa/onnx/vad/apk-cn.html +[apk-vad-asr]: https://k2-fsa.github.io/sherpa/onnx/vad/apk-asr.html +[apk-vad-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/vad/apk-asr-cn.html +[apk-2pass]: https://k2-fsa.github.io/sherpa/onnx/android/apk-2pass.html +[apk-2pass-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-2pass-cn.html +[apk-at]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk.html +[apk-at-cn]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk-cn.html +[apk-at-wearos]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk-wearos.html +[apk-at-wearos-cn]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk-wearos-cn.html +[apk-sid]: https://k2-fsa.github.io/sherpa/onnx/speaker-identification/apk.html +[apk-sid-cn]: https://k2-fsa.github.io/sherpa/onnx/speaker-identification/apk-cn.html +[apk-slid]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/apk.html +[apk-slid-cn]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/apk-cn.html +[apk-kws]: https://k2-fsa.github.io/sherpa/onnx/kws/apk.html +[apk-kws-cn]: https://k2-fsa.github.io/sherpa/onnx/kws/apk-cn.html +[apk-flutter-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/flutter/pre-built-app.html#streaming-speech-recognition-stt-asr +[apk-flutter-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/pre-built-app.html#streaming-speech-recognition-stt-asr +[flutter-tts-android]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-android.html +[flutter-tts-android-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-android-cn.html +[flutter-tts-linux]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-linux.html +[flutter-tts-linux-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-linux-cn.html +[flutter-tts-macos-x64]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-x64.html +[flutter-tts-macos-arm64-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-arm64-cn.html +[flutter-tts-macos-arm64]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-arm64.html +[flutter-tts-macos-x64-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-x64-cn.html +[flutter-tts-win-x64]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-win.html +[flutter-tts-win-x64-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-win-cn.html +[lazarus-subtitle]: https://k2-fsa.github.io/sherpa/onnx/lazarus/download-generated-subtitles.html +[lazarus-subtitle-cn]: https://k2-fsa.github.io/sherpa/onnx/lazarus/download-generated-subtitles-cn.html +[asr-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models +[tts-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +[vad-models]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +[kws-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/kws-models +[at-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/audio-tagging-models +[sid-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +[slid-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +[punct-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/punctuation-models +[speaker-segmentation-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +[GigaSpeech]: https://github.com/SpeechColab/GigaSpeech +[WenetSpeech]: https://github.com/wenet-e2e/WenetSpeech +[sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +[sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16.tar.bz2 +[sherpa-onnx-streaming-zipformer-korean-2024-06-16]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-korean-2024-06-16.tar.bz2 +[sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2 +[sherpa-onnx-streaming-zipformer-en-20M-2023-02-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 +[sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01.tar.bz2 +[sherpa-onnx-zipformer-ru-2024-09-18]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ru-2024-09-18.tar.bz2 +[sherpa-onnx-zipformer-korean-2024-06-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-korean-2024-06-24.tar.bz2 +[sherpa-onnx-zipformer-thai-2024-06-20]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-thai-2024-06-20.tar.bz2 +[sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24.tar.bz2 +[sherpa-onnx-paraformer-zh-2024-03-09]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2024-03-09.tar.bz2 +[sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24.tar.bz2 +[sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04.tar.bz2 +[sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +[sherpa-onnx-streaming-zipformer-fr-2023-04-14]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-fr-2023-04-14.tar.bz2 +[Moonshine tiny]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +[NVIDIA Jetson Orin NX]: https://developer.download.nvidia.com/assets/embedded/secure/jetson/orin_nx/docs/Jetson_Orin_NX_DS-10712-001_v0.5.pdf?RCPGu9Q6OVAOv7a7vgtwc9-BLScXRIWq6cSLuditMALECJ_dOj27DgnqAPGVnT2VpiNpQan9SyFy-9zRykR58CokzbXwjSA7Gj819e91AXPrWkGZR3oS1VLxiDEpJa_Y0lr7UT-N4GnXtb8NlUkP4GkCkkF_FQivGPrAucCUywL481GH_WpP_p7ziHU1Wg==&t=eyJscyI6ImdzZW8iLCJsc2QiOiJodHRwczovL3d3dy5nb29nbGUuY29tLmhrLyJ9 +[NVIDIA Jetson Nano B01]: https://www.seeedstudio.com/blog/2020/01/16/new-revision-of-jetson-nano-dev-kit-now-supports-new-jetson-nano-module/ +[speech-enhancement-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models +[source-separation-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/source-separation-models +[RK3588]: https://www.rock-chips.com/uploads/pdf/2022.8.26/192/RK3588%20Brief%20Datasheet.pdf +[spleeter]: https://github.com/deezer/spleeter +[UVR]: https://github.com/Anjok07/ultimatevocalremovergui +[gtcrn]: https://github.com/Xiaobin-Rong/gtcrn +[tts-url]: https://k2-fsa.github.io/sherpa/onnx/tts/all-in-one.html +[ss-url]: https://k2-fsa.github.io/sherpa/onnx/source-separation/index.html +[sd-url]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/index.html +[slid-url]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/index.html +[at-url]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/index.html +[vad-url]: https://k2-fsa.github.io/sherpa/onnx/vad/index.html +[kws-url]: https://k2-fsa.github.io/sherpa/onnx/kws/index.html +[punct-url]: https://k2-fsa.github.io/sherpa/onnx/punctuation/index.html +[se-url]: https://k2-fsa.github.io/sherpa/onnx/speech-enhancement/index.html +[rknpu-doc]: https://k2-fsa.github.io/sherpa/onnx/rknn/index.html +[qnn-doc]: https://k2-fsa.github.io/sherpa/onnx/qnn/index.html +[ascend-doc]: https://k2-fsa.github.io/sherpa/onnx/ascend/index.html +[axera-npu]: https://axera-tech.com/Skill/166.html +[SpacemiT-K1]: https://cdn-resource.spacemit.com/file/chip/K1/K1_brief_zh.pdf +[SpacemiT-K3]: https://cdn-resource.spacemit.com/file/chip/K3/K3_brief_zh.pdf diff --git a/crates/sherpa-onnx-sys/build.rs b/crates/sherpa-onnx-sys/build.rs new file mode 100644 index 0000000000..274dfb293e --- /dev/null +++ b/crates/sherpa-onnx-sys/build.rs @@ -0,0 +1,475 @@ +use std::env; +use std::error::Error; +use std::ffi::OsStr; +use std::fs; +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; +use std::{collections::HashSet, ffi::OsString}; + +use bzip2::read::BzDecoder; +use tar::Archive; + +const RELEASE_BASE_URL: &str = "https://github.com/k2-fsa/sherpa-onnx/releases/download"; +const SHERPA_ONNX_STATIC_LIBS: &[&str] = &[ + "sherpa-onnx-c-api", + "sherpa-onnx-core", + "kaldi-decoder-core", + "sherpa-onnx-kaldifst-core", + "sherpa-onnx-fstfar", + "sherpa-onnx-fst", + "kaldi-native-fbank-core", + "kissfft-float", + "piper_phonemize", + "espeak-ng", + "ucd", + "onnxruntime", + "ssentencepiece_core", +]; + +type DynError = Box; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LinkMode { + Static, + Shared, +} + +fn main() { + if let Err(err) = try_main() { + panic!("{err}"); + } +} + +fn try_main() -> Result<(), DynError> { + println!("cargo:rerun-if-env-changed=SHERPA_ONNX_LIB_DIR"); + println!("cargo:rerun-if-env-changed=SHERPA_ONNX_ARCHIVE_DIR"); + println!("cargo:rerun-if-env-changed=DOCS_RS"); + + if env::var_os("DOCS_RS").is_some() { + // docs.rs sets DOCS_RS=1; skip downloading/linking native libraries + // so that `cargo doc` can succeed without the real C artifacts. + return Ok(()); + } + + let target_os = env::var("CARGO_CFG_TARGET_OS")?; + let target_arch = env::var("CARGO_CFG_TARGET_ARCH")?; + let link_mode = resolve_link_mode()?; + let lib_dir = resolve_lib_dir(link_mode, &target_os, &target_arch)?; + + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + + if link_mode == LinkMode::Shared && matches!(target_os.as_str(), "linux" | "macos") { + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display()); + emit_relative_rpath(&target_os); + copy_unix_runtime_libs(&lib_dir, &target_os)?; + } + + if link_mode == LinkMode::Shared && target_os == "windows" { + copy_windows_runtime_dlls(&lib_dir)?; + } + + match (link_mode, target_os.as_str()) { + (LinkMode::Static, "ios") => emit_ios_link_directives(), + (LinkMode::Shared, "android") => emit_android_link_directives(), + (LinkMode::Static, _) => emit_static_link_directives(&target_os), + (LinkMode::Shared, _) => emit_shared_link_directives(), + } + + Ok(()) +} + +fn resolve_link_mode() -> Result { + let static_enabled = env::var_os("CARGO_FEATURE_STATIC").is_some(); + let shared_enabled = env::var_os("CARGO_FEATURE_SHARED").is_some(); + + if static_enabled && shared_enabled { + return Err("Features `static` and `shared` cannot be enabled at the same time".into()); + } + + if shared_enabled { + Ok(LinkMode::Shared) + } else { + Ok(LinkMode::Static) + } +} + +fn resolve_lib_dir( + link_mode: LinkMode, + target_os: &str, + target_arch: &str, +) -> Result { + if let Some(path) = env::var_os("SHERPA_ONNX_LIB_DIR") { + let path = PathBuf::from(path); + if !path.is_dir() { + return Err(format!( + "SHERPA_ONNX_LIB_DIR does not exist or is not a directory: {}", + path.display() + ) + .into()); + } + return Ok(path); + } + + if matches!(target_os, "android" | "ios") { + return Err(format!( + "SHERPA_ONNX_LIB_DIR must point at the verified sherpa-onnx v{} \ + mobile libraries when building for os={target_os}, arch={target_arch}", + env!("CARGO_PKG_VERSION") + ) + .into()); + } + + download_prebuilt_libs(link_mode, target_os, target_arch) +} + +fn download_prebuilt_libs( + link_mode: LinkMode, + target_os: &str, + target_arch: &str, +) -> Result { + let archive_name = archive_name(link_mode, target_os, target_arch)?; + let archive_stem = archive_name.trim_end_matches(".tar.bz2"); + + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + let cache_root = target_dir_from_out_dir(&out_dir)?.join("sherpa-onnx-prebuilt"); + let extracted_dir = cache_root.join(archive_stem); + let lib_dir = extracted_dir.join("lib"); + + if lib_dir.is_dir() { + return Ok(lib_dir); + } + + fs::create_dir_all(&cache_root)?; + + let archive_path = cache_root.join(&archive_name); + if !archive_path.is_file() { + if let Some(local_archive_dir) = env::var_os("SHERPA_ONNX_ARCHIVE_DIR") { + let local_archive_path = PathBuf::from(local_archive_dir).join(&archive_name); + if !local_archive_path.is_file() { + return Err(format!( + "SHERPA_ONNX_ARCHIVE_DIR does not contain expected archive: {}", + local_archive_path.display() + ) + .into()); + } + + copy_file_atomically(&local_archive_path, &archive_path)?; + } else { + let version = env!("CARGO_PKG_VERSION"); + let url = format!("{RELEASE_BASE_URL}/v{version}/{archive_name}"); + eprintln!("Downloading sherpa-onnx libs from {url}"); + + let response = ureq::builder() + .try_proxy_from_env(true) + .build() + .get(&url) + .call() + .map_err(|e| format!("Failed to download sherpa-onnx archive from {url}: {e}"))?; + let mut reader = response.into_reader(); + write_reader_atomically(&mut reader, &archive_path)?; + } + } + + if extracted_dir.exists() { + fs::remove_dir_all(&extracted_dir)?; + } + + let unpack_result: Result<(), DynError> = (|| { + let tar_file = File::open(&archive_path)?; + let decoder = BzDecoder::new(tar_file); + let mut archive = Archive::new(decoder); + archive.unpack(&cache_root)?; + Ok(()) + })(); + if let Err(err) = unpack_result { + let _ = fs::remove_file(&archive_path); + let _ = fs::remove_dir_all(&extracted_dir); + return Err(format!( + "Failed to unpack cached archive {}: {err}", + archive_path.display() + ) + .into()); + } + + if !lib_dir.is_dir() { + return Err(format!( + "Downloaded archive did not contain a lib directory: {}", + lib_dir.display() + ) + .into()); + } + + eprintln!("Downloaded sherpa-onnx libs to {}", extracted_dir.display()); + + Ok(lib_dir) +} + +fn archive_name( + link_mode: LinkMode, + target_os: &str, + target_arch: &str, +) -> Result { + let version = env!("CARGO_PKG_VERSION"); + let name = match (link_mode, target_os, target_arch) { + (LinkMode::Static, "linux", "x86_64") => { + format!("sherpa-onnx-v{version}-linux-x64-static-lib.tar.bz2") + } + (LinkMode::Static, "linux", "aarch64") => { + format!("sherpa-onnx-v{version}-linux-aarch64-static-lib.tar.bz2") + } + (LinkMode::Static, "macos", "x86_64") => { + format!("sherpa-onnx-v{version}-osx-x64-static-lib.tar.bz2") + } + (LinkMode::Static, "macos", "aarch64") => { + format!("sherpa-onnx-v{version}-osx-arm64-static-lib.tar.bz2") + } + (LinkMode::Static, "windows", "x86_64") => { + format!("sherpa-onnx-v{version}-win-x64-static-MT-Release-lib.tar.bz2") + } + (LinkMode::Shared, "linux", "x86_64") => { + format!("sherpa-onnx-v{version}-linux-x64-shared-lib.tar.bz2") + } + (LinkMode::Shared, "linux", "aarch64") => { + format!("sherpa-onnx-v{version}-linux-aarch64-shared-cpu-lib.tar.bz2") + } + (LinkMode::Shared, "macos", "x86_64") => { + format!("sherpa-onnx-v{version}-osx-x64-shared-lib.tar.bz2") + } + (LinkMode::Shared, "macos", "aarch64") => { + format!("sherpa-onnx-v{version}-osx-arm64-shared-lib.tar.bz2") + } + (LinkMode::Shared, "windows", "x86_64") => { + format!("sherpa-onnx-v{version}-win-x64-shared-MT-Release-lib.tar.bz2") + } + _ => { + return Err(format!( + "Unsupported target for sherpa-onnx prebuilt libs: os={target_os}, arch={target_arch}" + ) + .into()) + } + }; + + Ok(name) +} + +fn emit_shared_link_directives() { + println!("cargo:rustc-link-lib=dylib=sherpa-onnx-c-api"); + println!("cargo:rustc-link-lib=dylib=onnxruntime"); +} + +fn emit_android_link_directives() { + println!("cargo:rustc-link-lib=dylib=sherpa-onnx-c-api"); + println!("cargo:rustc-link-lib=dylib=onnxruntime"); + println!("cargo:rustc-link-lib=dylib=log"); +} + +fn emit_ios_link_directives() { + println!("cargo:rustc-link-lib=static=sherpa-onnx"); + println!("cargo:rustc-link-lib=static=onnxruntime"); + println!("cargo:rustc-link-lib=dylib=c++"); + for framework in [ + "Accelerate", + "CoreML", + "Foundation", + "Metal", + "QuartzCore", + "Security", + ] { + println!("cargo:rustc-link-lib=framework={framework}"); + } +} + +fn emit_static_link_directives(target_os: &str) { + for lib in SHERPA_ONNX_STATIC_LIBS { + println!("cargo:rustc-link-lib=static={lib}"); + } + + match target_os { + "linux" => { + println!("cargo:rustc-link-lib=dylib=stdc++"); + println!("cargo:rustc-link-lib=dylib=m"); + println!("cargo:rustc-link-lib=dylib=pthread"); + println!("cargo:rustc-link-lib=dylib=dl"); + } + "macos" => { + println!("cargo:rustc-link-lib=dylib=c++"); + println!("cargo:rustc-link-lib=framework=Foundation"); + } + _ => {} + } +} + +fn target_dir_from_out_dir(out_dir: &Path) -> Result { + if let Ok(explicit_target_dir) = env::var("CARGO_TARGET_DIR") { + return Ok(PathBuf::from(explicit_target_dir)); + } + + if let Some(target_dir) = out_dir + .ancestors() + .find(|path| path.file_name() == Some(OsStr::new("target"))) + { + return Ok(target_dir.to_path_buf()); + } + + Ok(out_dir.to_path_buf()) +} + +fn emit_relative_rpath(target_os: &str) { + match target_os { + "linux" => println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN"), + "macos" => println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path"), + _ => {} + } +} + +fn profile_output_dirs() -> Result<[PathBuf; 2], DynError> { + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + let profile = env::var("PROFILE")?; + let profile_dir = out_dir + .ancestors() + .find(|path| path.file_name() == Some(OsStr::new(&profile))) + .ok_or_else(|| { + format!( + "Could not locate Cargo profile directory from {}", + out_dir.display() + ) + })? + .to_path_buf(); + + Ok([profile_dir.clone(), profile_dir.join("examples")]) +} + +fn copy_unix_runtime_libs(lib_dir: &Path, target_os: &str) -> Result<(), DynError> { + let runtime_libs: Vec = fs::read_dir(lib_dir)? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| { + path.file_name() + .and_then(OsStr::to_str) + .map(|name| match target_os { + "linux" => name.contains(".so"), + "macos" => name.ends_with(".dylib"), + _ => false, + }) + .unwrap_or(false) + }) + .collect(); + + if runtime_libs.is_empty() { + return Err(format!("No shared runtime libraries found in {}", lib_dir.display()).into()); + } + + let mut copy_plan = Vec::<(PathBuf, OsString)>::new(); + let mut planned_names = HashSet::::new(); + + for lib in runtime_libs { + if !lib.exists() { + continue; + } + + let lib_name = lib + .file_name() + .ok_or_else(|| format!("Invalid runtime library path: {}", lib.display()))? + .to_os_string(); + + let source = fs::canonicalize(&lib).unwrap_or(lib.clone()); + if planned_names.insert(lib_name.clone()) { + copy_plan.push((source.clone(), lib_name)); + } + + if let Some(source_name) = source.file_name() { + let source_name = source_name.to_os_string(); + if planned_names.insert(source_name.clone()) { + copy_plan.push((source.clone(), source_name)); + } + } + } + + if copy_plan.is_empty() { + return Err(format!( + "No usable shared runtime libraries found in {}", + lib_dir.display() + ) + .into()); + } + + for dest_dir in profile_output_dirs()? { + fs::create_dir_all(&dest_dir)?; + for (source, dest_name) in ©_plan { + let dest = dest_dir.join(dest_name); + fs::copy(source, &dest)?; + } + } + + Ok(()) +} + +fn temp_path_for(path: &Path) -> PathBuf { + let mut temp_name = path + .file_name() + .map(OsStr::to_os_string) + .unwrap_or_else(|| OsString::from("tmp")); + temp_name.push(".part"); + path.with_file_name(temp_name) +} + +fn copy_file_atomically(src: &Path, dst: &Path) -> Result<(), DynError> { + let temp_path = temp_path_for(dst); + if temp_path.exists() { + let _ = fs::remove_file(&temp_path); + } + fs::copy(src, &temp_path)?; + fs::rename(&temp_path, dst)?; + Ok(()) +} + +fn write_reader_atomically(reader: &mut dyn io::Read, dst: &Path) -> Result<(), DynError> { + let temp_path = temp_path_for(dst); + if temp_path.exists() { + let _ = fs::remove_file(&temp_path); + } + + { + let mut file = File::create(&temp_path)?; + io::copy(reader, &mut file)?; + file.sync_all()?; + } + + fs::rename(&temp_path, dst)?; + Ok(()) +} + +fn copy_windows_runtime_dlls(lib_dir: &Path) -> Result<(), DynError> { + let dlls: Vec = fs::read_dir(lib_dir)? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| path.extension() == Some(OsStr::new("dll"))) + .collect(); + + if dlls.is_empty() { + println!( + "cargo:warning=No runtime DLLs found in {}", + lib_dir.display() + ); + return Ok(()); + } + + let [profile_dir, examples_dir] = profile_output_dirs()?; + for dest_dir in [profile_dir.clone(), examples_dir] { + fs::create_dir_all(&dest_dir)?; + for dll in &dlls { + let dest = dest_dir.join( + dll.file_name() + .ok_or_else(|| format!("Invalid DLL path: {}", dll.display()))?, + ); + fs::copy(dll, &dest)?; + } + } + + println!( + "cargo:warning=Copied Windows runtime DLLs to {} and {}/examples", + profile_dir.display(), + profile_dir.display() + ); + + Ok(()) +} diff --git a/crates/sherpa-onnx-sys/src/audio_tagging.rs b/crates/sherpa-onnx-sys/src/audio_tagging.rs new file mode 100644 index 0000000000..6edf2d5d0a --- /dev/null +++ b/crates/sherpa-onnx-sys/src/audio_tagging.rs @@ -0,0 +1,60 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineZipformerAudioTaggingModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct AudioTaggingModelConfig { + pub zipformer: OfflineZipformerAudioTaggingModelConfig, + pub ced: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct AudioTaggingConfig { + pub model: AudioTaggingModelConfig, + pub labels: *const c_char, + pub top_k: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct AudioEvent { + pub name: *const c_char, + pub index: i32, + pub prob: c_float, +} + +#[repr(C)] +pub struct AudioTagging { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateAudioTagging(config: *const AudioTaggingConfig) -> *const AudioTagging; + + pub fn SherpaOnnxDestroyAudioTagging(tagger: *const AudioTagging); + + pub fn SherpaOnnxAudioTaggingCreateOfflineStream( + tagger: *const AudioTagging, + ) -> *const crate::offline_asr::OfflineStream; + + pub fn SherpaOnnxAudioTaggingCompute( + tagger: *const AudioTagging, + s: *const crate::offline_asr::OfflineStream, + top_k: i32, + ) -> *const *const AudioEvent; + + pub fn SherpaOnnxAudioTaggingFreeResults(p: *const *const AudioEvent); +} diff --git a/crates/sherpa-onnx-sys/src/kws.rs b/crates/sherpa-onnx-sys/src/kws.rs new file mode 100644 index 0000000000..25a2dd26c8 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/kws.rs @@ -0,0 +1,88 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KeywordResult { + pub keyword: *const c_char, + pub tokens: *const c_char, + pub tokens_arr: *const *const c_char, + pub count: i32, + pub timestamps: *mut c_float, + pub start_time: c_float, + pub json: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KeywordSpotterConfig { + pub feat_config: super::online_asr::FeatureConfig, + pub model_config: super::online_asr::OnlineModelConfig, + pub max_active_paths: i32, + pub num_trailing_blanks: i32, + pub keywords_score: c_float, + pub keywords_threshold: c_float, + pub keywords_file: *const c_char, + pub keywords_buf: *const c_char, + pub keywords_buf_size: i32, +} + +#[repr(C)] +pub struct KeywordSpotter { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateKeywordSpotter( + config: *const KeywordSpotterConfig, + ) -> *const KeywordSpotter; + + pub fn SherpaOnnxDestroyKeywordSpotter(spotter: *const KeywordSpotter); + + pub fn SherpaOnnxCreateKeywordStream( + spotter: *const KeywordSpotter, + ) -> *const super::online_asr::OnlineStream; + + pub fn SherpaOnnxCreateKeywordStreamWithKeywords( + spotter: *const KeywordSpotter, + keywords: *const c_char, + ) -> *const super::online_asr::OnlineStream; + + pub fn SherpaOnnxIsKeywordStreamReady( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ) -> i32; + + pub fn SherpaOnnxDecodeKeywordStream( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ); + + pub fn SherpaOnnxResetKeywordStream( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ); + + pub fn SherpaOnnxDecodeMultipleKeywordStreams( + spotter: *const KeywordSpotter, + streams: *const *const super::online_asr::OnlineStream, + n: i32, + ); + + pub fn SherpaOnnxGetKeywordResult( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ) -> *const KeywordResult; + + pub fn SherpaOnnxDestroyKeywordResult(r: *const KeywordResult); + + pub fn SherpaOnnxGetKeywordResultAsJson( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ) -> *const c_char; + + pub fn SherpaOnnxFreeKeywordResultJson(s: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/lib.rs b/crates/sherpa-onnx-sys/src/lib.rs new file mode 100644 index 0000000000..b4161cc616 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/lib.rs @@ -0,0 +1,42 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +extern "C" { + pub fn SherpaOnnxGetVersionStr() -> *const c_char; + pub fn SherpaOnnxGetGitSha1() -> *const c_char; + pub fn SherpaOnnxGetGitDate() -> *const c_char; + pub fn SherpaOnnxFileExists(filename: *const c_char) -> i32; +} + +pub mod audio_tagging; +pub mod kws; +pub mod offline_asr; +pub mod offline_punctuation; +pub mod offline_speaker_diarization; +pub mod online_asr; +pub mod online_punctuation; +pub mod resampler; +pub mod speaker_embedding; +pub mod speech_denoiser; +pub mod spoken_language_identification; +pub mod tts; +pub mod vad; +pub mod wave; + +pub use audio_tagging::*; +pub use kws::*; +pub use offline_asr::*; +pub use offline_punctuation::*; +pub use offline_speaker_diarization::*; +pub use online_asr::*; +pub use online_punctuation::*; +pub use resampler::*; +pub use speaker_embedding::*; +pub use speech_denoiser::*; +pub use spoken_language_identification::*; +pub use tts::*; +pub use vad::*; +pub use wave::*; diff --git a/crates/sherpa-onnx-sys/src/offline_asr.rs b/crates/sherpa-onnx-sys/src/offline_asr.rs new file mode 100644 index 0000000000..9d0d28c748 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/offline_asr.rs @@ -0,0 +1,281 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTransducerModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub joiner: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineParaformerModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineNemoEncDecCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineWhisperModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub language: *const c_char, + pub task: *const c_char, + pub tail_paddings: i32, + pub enable_token_timestamps: i32, + pub enable_segment_timestamps: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineCanaryModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub src_lang: *const c_char, + pub tgt_lang: *const c_char, + pub use_pnc: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineFireRedAsrModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineMoonshineModelConfig { + pub preprocessor: *const c_char, + pub encoder: *const c_char, + pub uncached_decoder: *const c_char, + pub cached_decoder: *const c_char, + pub merged_decoder: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTdnnModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineLMConfig { + pub model: *const c_char, + pub scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSenseVoiceModelConfig { + pub model: *const c_char, + pub language: *const c_char, + pub use_itn: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineDolphinModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineZipformerCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineWenetCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineOmnilingualAsrCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineFunASRNanoModelConfig { + pub encoder_adaptor: *const c_char, + pub llm: *const c_char, + pub embedding: *const c_char, + pub tokenizer: *const c_char, + pub system_prompt: *const c_char, + pub user_prompt: *const c_char, + pub max_new_tokens: i32, + pub temperature: c_float, + pub top_p: c_float, + pub seed: i32, + pub language: *const c_char, + pub itn: i32, + pub hotwords: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineMedAsrCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineFireRedAsrCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineQwen3ASRModelConfig { + pub conv_frontend: *const c_char, + pub encoder: *const c_char, + pub decoder: *const c_char, + pub tokenizer: *const c_char, + pub max_total_len: i32, + pub max_new_tokens: i32, + pub temperature: c_float, + pub top_p: c_float, + pub seed: i32, + pub hotwords: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineCohereTranscribeModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub language: *const c_char, + pub use_punct: i32, + pub use_itn: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineModelConfig { + pub transducer: OfflineTransducerModelConfig, + pub paraformer: OfflineParaformerModelConfig, + pub nemo_ctc: OfflineNemoEncDecCtcModelConfig, + pub whisper: OfflineWhisperModelConfig, + pub tdnn: OfflineTdnnModelConfig, + + pub tokens: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, + pub model_type: *const c_char, + pub modeling_unit: *const c_char, + pub bpe_vocab: *const c_char, + pub telespeech_ctc: *const c_char, + + pub sense_voice: OfflineSenseVoiceModelConfig, + pub moonshine: OfflineMoonshineModelConfig, + pub fire_red_asr: OfflineFireRedAsrModelConfig, + pub dolphin: OfflineDolphinModelConfig, + pub zipformer_ctc: OfflineZipformerCtcModelConfig, + pub canary: OfflineCanaryModelConfig, + pub wenet_ctc: OfflineWenetCtcModelConfig, + pub omnilingual: OfflineOmnilingualAsrCtcModelConfig, + pub medasr: OfflineMedAsrCtcModelConfig, + pub funasr_nano: OfflineFunASRNanoModelConfig, + pub fire_red_asr_ctc: OfflineFireRedAsrCtcModelConfig, + pub qwen3_asr: OfflineQwen3ASRModelConfig, + pub cohere_transcribe: OfflineCohereTranscribeModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineRecognizerConfig { + pub feat_config: super::online_asr::FeatureConfig, + pub model_config: OfflineModelConfig, + pub lm_config: OfflineLMConfig, + + pub decoding_method: *const c_char, + pub max_active_paths: i32, + pub hotwords_file: *const c_char, + pub hotwords_score: c_float, + pub rule_fsts: *const c_char, + pub rule_fars: *const c_char, + pub blank_penalty: c_float, + pub hr: super::online_asr::HomophoneReplacerConfig, +} + +#[repr(C)] +pub struct OfflineRecognizer { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OfflineStream { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineRecognizer( + config: *const OfflineRecognizerConfig, + ) -> *const OfflineRecognizer; + + pub fn SherpaOnnxDestroyOfflineRecognizer(recognizer: *const OfflineRecognizer); + + pub fn SherpaOnnxCreateOfflineStream( + recognizer: *const OfflineRecognizer, + ) -> *const OfflineStream; + + pub fn SherpaOnnxCreateOfflineStreamWithHotwords( + recognizer: *const OfflineRecognizer, + hotwords: *const c_char, + ) -> *const OfflineStream; + + pub fn SherpaOnnxDestroyOfflineStream(stream: *const OfflineStream); + + pub fn SherpaOnnxAcceptWaveformOffline( + stream: *const OfflineStream, + sample_rate: i32, + samples: *const f32, + n: i32, + ); + + pub fn SherpaOnnxOfflineStreamSetOption( + stream: *const OfflineStream, + key: *const c_char, + value: *const c_char, + ); + + pub fn SherpaOnnxOfflineStreamGetOption( + stream: *const OfflineStream, + key: *const c_char, + ) -> *const c_char; + + pub fn SherpaOnnxOfflineStreamHasOption( + stream: *const OfflineStream, + key: *const c_char, + ) -> i32; + + pub fn SherpaOnnxDecodeOfflineStream( + recognizer: *const OfflineRecognizer, + stream: *const OfflineStream, + ); + + pub fn SherpaOnnxDecodeMultipleOfflineStreams( + recognizer: *const OfflineRecognizer, + streams: *const *const OfflineStream, + n: i32, + ); + + pub fn SherpaOnnxGetOfflineStreamResultAsJson(stream: *const OfflineStream) -> *const c_char; + + pub fn SherpaOnnxDestroyOfflineStreamResultJson(s: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/offline_punctuation.rs b/crates/sherpa-onnx-sys/src/offline_punctuation.rs new file mode 100644 index 0000000000..06547846cf --- /dev/null +++ b/crates/sherpa-onnx-sys/src/offline_punctuation.rs @@ -0,0 +1,40 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflinePunctuationModelConfig { + pub ct_transformer: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflinePunctuationConfig { + pub model: OfflinePunctuationModelConfig, +} + +#[repr(C)] +pub struct OfflinePunctuation { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflinePunctuation( + config: *const OfflinePunctuationConfig, + ) -> *const OfflinePunctuation; + + pub fn SherpaOnnxDestroyOfflinePunctuation(punct: *const OfflinePunctuation); + + pub fn SherpaOfflinePunctuationAddPunct( + punct: *const OfflinePunctuation, + text: *const c_char, + ) -> *const c_char; + + pub fn SherpaOfflinePunctuationFreeText(text: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs b/crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs new file mode 100644 index 0000000000..00232e0a98 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs @@ -0,0 +1,98 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerSegmentationPyannoteModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerSegmentationModelConfig { + pub pyannote: OfflineSpeakerSegmentationPyannoteModelConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct FastClusteringConfig { + pub num_clusters: i32, + pub threshold: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerDiarizationConfig { + pub segmentation: OfflineSpeakerSegmentationModelConfig, + pub embedding: crate::speaker_embedding::SpeakerEmbeddingExtractorConfig, + pub clustering: FastClusteringConfig, + pub min_duration_on: c_float, + pub min_duration_off: c_float, +} + +#[repr(C)] +pub struct OfflineSpeakerDiarization { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OfflineSpeakerDiarizationResult { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerDiarizationSegment { + pub start: c_float, + pub end: c_float, + pub speaker: i32, +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineSpeakerDiarization( + config: *const OfflineSpeakerDiarizationConfig, + ) -> *const OfflineSpeakerDiarization; + + pub fn SherpaOnnxDestroyOfflineSpeakerDiarization(sd: *const OfflineSpeakerDiarization); + + pub fn SherpaOnnxOfflineSpeakerDiarizationGetSampleRate( + sd: *const OfflineSpeakerDiarization, + ) -> i32; + + pub fn SherpaOnnxOfflineSpeakerDiarizationSetConfig( + sd: *const OfflineSpeakerDiarization, + config: *const OfflineSpeakerDiarizationConfig, + ); + + pub fn SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers( + r: *const OfflineSpeakerDiarizationResult, + ) -> i32; + + pub fn SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments( + r: *const OfflineSpeakerDiarizationResult, + ) -> i32; + + pub fn SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime( + r: *const OfflineSpeakerDiarizationResult, + ) -> *const OfflineSpeakerDiarizationSegment; + + pub fn SherpaOnnxOfflineSpeakerDiarizationDestroySegment( + s: *const OfflineSpeakerDiarizationSegment, + ); + + pub fn SherpaOnnxOfflineSpeakerDiarizationProcess( + sd: *const OfflineSpeakerDiarization, + samples: *const c_float, + n: i32, + ) -> *const OfflineSpeakerDiarizationResult; + + pub fn SherpaOnnxOfflineSpeakerDiarizationDestroyResult( + r: *const OfflineSpeakerDiarizationResult, + ); +} diff --git a/crates/sherpa-onnx-sys/src/online_asr.rs b/crates/sherpa-onnx-sys/src/online_asr.rs new file mode 100644 index 0000000000..522f934d79 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/online_asr.rs @@ -0,0 +1,202 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineTransducerModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub joiner: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineParaformerModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineZipformer2CtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineNemoCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineToneCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineModelConfig { + pub transducer: OnlineTransducerModelConfig, + pub paraformer: OnlineParaformerModelConfig, + pub zipformer2_ctc: OnlineZipformer2CtcModelConfig, + + pub tokens: *const c_char, + pub num_threads: i32, + pub provider: *const c_char, + pub debug: i32, + + pub model_type: *const c_char, + + // cjkchar | bpe | cjkchar+bpe + pub modeling_unit: *const c_char, + + pub bpe_vocab: *const c_char, + + pub tokens_buf: *const u8, + pub tokens_buf_size: i32, + + pub nemo_ctc: OnlineNemoCtcModelConfig, + pub t_one_ctc: OnlineToneCtcModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct FeatureConfig { + pub sample_rate: i32, + pub feature_dim: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineCtcFstDecoderConfig { + pub graph: *const c_char, + pub max_active: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct HomophoneReplacerConfig { + pub dict_dir: *const c_char, + pub lexicon: *const c_char, + pub rule_fsts: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineRecognizerConfig { + pub feat_config: FeatureConfig, + pub model_config: OnlineModelConfig, + + // greedy_search | modified_beam_search + pub decoding_method: *const c_char, + + pub max_active_paths: i32, + + pub enable_endpoint: i32, + + pub rule1_min_trailing_silence: c_float, + pub rule2_min_trailing_silence: c_float, + pub rule3_min_utterance_length: c_float, + + pub hotwords_file: *const c_char, + pub hotwords_score: c_float, + + pub ctc_fst_decoder_config: OnlineCtcFstDecoderConfig, + + pub rule_fsts: *const c_char, + pub rule_fars: *const c_char, + + pub blank_penalty: c_float, + + pub hotwords_buf: *const u8, + pub hotwords_buf_size: i32, + + pub hr: HomophoneReplacerConfig, +} + +#[repr(C)] +pub struct OnlineRecognizer { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OnlineStream { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOnlineRecognizer( + config: *const OnlineRecognizerConfig, + ) -> *const OnlineRecognizer; + + pub fn SherpaOnnxDestroyOnlineRecognizer(recognizer: *const OnlineRecognizer); + + pub fn SherpaOnnxCreateOnlineStream(recognizer: *const OnlineRecognizer) + -> *const OnlineStream; + + pub fn SherpaOnnxCreateOnlineStreamWithHotwords( + recognizer: *const OnlineRecognizer, + hotwords: *const c_char, + ) -> *const OnlineStream; + + pub fn SherpaOnnxDestroyOnlineStream(stream: *const OnlineStream); + + pub fn SherpaOnnxOnlineStreamAcceptWaveform( + stream: *const OnlineStream, + sample_rate: i32, + samples: *const f32, + n: i32, + ); + + pub fn SherpaOnnxIsOnlineStreamReady( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ) -> i32; + + pub fn SherpaOnnxDecodeOnlineStream( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ); + + pub fn SherpaOnnxDecodeMultipleOnlineStreams( + recognizer: *const OnlineRecognizer, + streams: *const *const OnlineStream, + n: i32, + ); + + pub fn SherpaOnnxGetOnlineStreamResultAsJson( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ) -> *const c_char; + + pub fn SherpaOnnxDestroyOnlineStreamResultJson(s: *const c_char); + + pub fn SherpaOnnxOnlineStreamReset( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ); + + pub fn SherpaOnnxOnlineStreamInputFinished(stream: *const OnlineStream); + + pub fn SherpaOnnxOnlineStreamSetOption( + stream: *const OnlineStream, + key: *const c_char, + value: *const c_char, + ); + + pub fn SherpaOnnxOnlineStreamGetOption( + stream: *const OnlineStream, + key: *const c_char, + ) -> *const c_char; + + pub fn SherpaOnnxOnlineStreamHasOption(stream: *const OnlineStream, key: *const c_char) -> i32; + + pub fn SherpaOnnxOnlineStreamIsEndpoint( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ) -> i32; +} diff --git a/crates/sherpa-onnx-sys/src/online_punctuation.rs b/crates/sherpa-onnx-sys/src/online_punctuation.rs new file mode 100644 index 0000000000..1ff1fce1b0 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/online_punctuation.rs @@ -0,0 +1,41 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlinePunctuationModelConfig { + pub cnn_bilstm: *const c_char, + pub bpe_vocab: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlinePunctuationConfig { + pub model: OnlinePunctuationModelConfig, +} + +#[repr(C)] +pub struct OnlinePunctuation { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOnlinePunctuation( + config: *const OnlinePunctuationConfig, + ) -> *const OnlinePunctuation; + + pub fn SherpaOnnxDestroyOnlinePunctuation(punctuation: *const OnlinePunctuation); + + pub fn SherpaOnnxOnlinePunctuationAddPunct( + punctuation: *const OnlinePunctuation, + text: *const c_char, + ) -> *const c_char; + + pub fn SherpaOnnxOnlinePunctuationFreeText(text: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/resampler.rs b/crates/sherpa-onnx-sys/src/resampler.rs new file mode 100644 index 0000000000..fd49835662 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/resampler.rs @@ -0,0 +1,42 @@ +use std::os::raw::c_float; + +#[repr(C)] +pub struct SherpaOnnxLinearResampler { + _private: [u8; 0], +} + +#[repr(C)] +pub struct SherpaOnnxResampleOut { + pub samples: *const f32, + pub n: i32, +} + +extern "C" { + pub fn SherpaOnnxCreateLinearResampler( + samp_rate_in_hz: i32, + samp_rate_out_hz: i32, + filter_cutoff_hz: c_float, + num_zeros: i32, + ) -> *const SherpaOnnxLinearResampler; + + pub fn SherpaOnnxDestroyLinearResampler(p: *const SherpaOnnxLinearResampler); + + pub fn SherpaOnnxLinearResamplerReset(p: *const SherpaOnnxLinearResampler); + + pub fn SherpaOnnxLinearResamplerResample( + p: *const SherpaOnnxLinearResampler, + input: *const f32, + input_dim: i32, + flush: i32, + ) -> *const SherpaOnnxResampleOut; + + pub fn SherpaOnnxLinearResamplerResampleFree(p: *const SherpaOnnxResampleOut); + + pub fn SherpaOnnxLinearResamplerResampleGetInputSampleRate( + p: *const SherpaOnnxLinearResampler, + ) -> i32; + + pub fn SherpaOnnxLinearResamplerResampleGetOutputSampleRate( + p: *const SherpaOnnxLinearResampler, + ) -> i32; +} diff --git a/crates/sherpa-onnx-sys/src/speaker_embedding.rs b/crates/sherpa-onnx-sys/src/speaker_embedding.rs new file mode 100644 index 0000000000..ca745e3106 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/speaker_embedding.rs @@ -0,0 +1,131 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpeakerEmbeddingExtractorConfig { + pub model: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +pub struct SpeakerEmbeddingExtractor { + _private: [u8; 0], +} + +#[repr(C)] +pub struct SpeakerEmbeddingManager { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpeakerEmbeddingManagerSpeakerMatch { + pub score: c_float, + pub name: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpeakerEmbeddingManagerBestMatchesResult { + pub matches: *const SpeakerEmbeddingManagerSpeakerMatch, + pub count: i32, +} + +extern "C" { + pub fn SherpaOnnxCreateSpeakerEmbeddingExtractor( + config: *const SpeakerEmbeddingExtractorConfig, + ) -> *const SpeakerEmbeddingExtractor; + + pub fn SherpaOnnxDestroySpeakerEmbeddingExtractor(p: *const SpeakerEmbeddingExtractor); + + pub fn SherpaOnnxSpeakerEmbeddingExtractorDim(p: *const SpeakerEmbeddingExtractor) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorCreateStream( + p: *const SpeakerEmbeddingExtractor, + ) -> *const crate::online_asr::OnlineStream; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorIsReady( + p: *const SpeakerEmbeddingExtractor, + s: *const crate::online_asr::OnlineStream, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorComputeEmbedding( + p: *const SpeakerEmbeddingExtractor, + s: *const crate::online_asr::OnlineStream, + ) -> *const c_float; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorDestroyEmbedding(v: *const c_float); + + pub fn SherpaOnnxCreateSpeakerEmbeddingManager(dim: i32) -> *const SpeakerEmbeddingManager; + + pub fn SherpaOnnxDestroySpeakerEmbeddingManager(p: *const SpeakerEmbeddingManager); + + pub fn SherpaOnnxSpeakerEmbeddingManagerAdd( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const c_float, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerAddList( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const *const c_float, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerAddListFlattened( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const c_float, + n: i32, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerRemove( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerSearch( + p: *const SpeakerEmbeddingManager, + v: *const c_float, + threshold: c_float, + ) -> *const c_char; + + pub fn SherpaOnnxSpeakerEmbeddingManagerFreeSearch(name: *const c_char); + + pub fn SherpaOnnxSpeakerEmbeddingManagerGetBestMatches( + p: *const SpeakerEmbeddingManager, + v: *const c_float, + threshold: c_float, + n: i32, + ) -> *const SpeakerEmbeddingManagerBestMatchesResult; + + pub fn SherpaOnnxSpeakerEmbeddingManagerFreeBestMatches( + r: *const SpeakerEmbeddingManagerBestMatchesResult, + ); + + pub fn SherpaOnnxSpeakerEmbeddingManagerVerify( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const c_float, + threshold: c_float, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerContains( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerNumSpeakers(p: *const SpeakerEmbeddingManager) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerGetAllSpeakers( + p: *const SpeakerEmbeddingManager, + ) -> *const *const c_char; + + pub fn SherpaOnnxSpeakerEmbeddingManagerFreeAllSpeakers(names: *const *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/speech_denoiser.rs b/crates/sherpa-onnx-sys/src/speech_denoiser.rs new file mode 100644 index 0000000000..dcf7f1e0e4 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/speech_denoiser.rs @@ -0,0 +1,88 @@ +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserGtcrnModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserDpdfNetModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserModelConfig { + pub gtcrn: OfflineSpeechDenoiserGtcrnModelConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, + pub dpdfnet: OfflineSpeechDenoiserDpdfNetModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserConfig { + pub model: OfflineSpeechDenoiserModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineSpeechDenoiserConfig { + pub model: OfflineSpeechDenoiserModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct DenoisedAudio { + pub samples: *const f32, + pub n: i32, + pub sample_rate: i32, +} + +#[repr(C)] +pub struct OfflineSpeechDenoiser { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OnlineSpeechDenoiser { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineSpeechDenoiser( + config: *const OfflineSpeechDenoiserConfig, + ) -> *const OfflineSpeechDenoiser; + pub fn SherpaOnnxDestroyOfflineSpeechDenoiser(p: *const OfflineSpeechDenoiser); + pub fn SherpaOnnxOfflineSpeechDenoiserGetSampleRate(p: *const OfflineSpeechDenoiser) -> i32; + pub fn SherpaOnnxOfflineSpeechDenoiserRun( + p: *const OfflineSpeechDenoiser, + samples: *const f32, + n: i32, + sample_rate: i32, + ) -> *const DenoisedAudio; + + pub fn SherpaOnnxCreateOnlineSpeechDenoiser( + config: *const OnlineSpeechDenoiserConfig, + ) -> *const OnlineSpeechDenoiser; + pub fn SherpaOnnxDestroyOnlineSpeechDenoiser(p: *const OnlineSpeechDenoiser); + pub fn SherpaOnnxOnlineSpeechDenoiserGetSampleRate(p: *const OnlineSpeechDenoiser) -> i32; + pub fn SherpaOnnxOnlineSpeechDenoiserGetFrameShiftInSamples( + p: *const OnlineSpeechDenoiser, + ) -> i32; + pub fn SherpaOnnxOnlineSpeechDenoiserRun( + p: *const OnlineSpeechDenoiser, + samples: *const f32, + n: i32, + sample_rate: i32, + ) -> *const DenoisedAudio; + pub fn SherpaOnnxOnlineSpeechDenoiserFlush( + p: *const OnlineSpeechDenoiser, + ) -> *const DenoisedAudio; + pub fn SherpaOnnxOnlineSpeechDenoiserReset(p: *const OnlineSpeechDenoiser); + + pub fn SherpaOnnxDestroyDenoisedAudio(audio: *const DenoisedAudio); +} diff --git a/crates/sherpa-onnx-sys/src/spoken_language_identification.rs b/crates/sherpa-onnx-sys/src/spoken_language_identification.rs new file mode 100644 index 0000000000..f9d8f46753 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/spoken_language_identification.rs @@ -0,0 +1,54 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpokenLanguageIdentificationWhisperConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub tail_paddings: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpokenLanguageIdentificationConfig { + pub whisper: SpokenLanguageIdentificationWhisperConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +pub struct SpokenLanguageIdentification { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpokenLanguageIdentificationResult { + pub lang: *const c_char, +} + +extern "C" { + pub fn SherpaOnnxCreateSpokenLanguageIdentification( + config: *const SpokenLanguageIdentificationConfig, + ) -> *const SpokenLanguageIdentification; + + pub fn SherpaOnnxDestroySpokenLanguageIdentification(slid: *const SpokenLanguageIdentification); + + pub fn SherpaOnnxSpokenLanguageIdentificationCreateOfflineStream( + slid: *const SpokenLanguageIdentification, + ) -> *const super::offline_asr::OfflineStream; + + pub fn SherpaOnnxSpokenLanguageIdentificationCompute( + slid: *const SpokenLanguageIdentification, + stream: *const super::offline_asr::OfflineStream, + ) -> *const SpokenLanguageIdentificationResult; + + pub fn SherpaOnnxDestroySpokenLanguageIdentificationResult( + r: *const SpokenLanguageIdentificationResult, + ); +} diff --git a/crates/sherpa-onnx-sys/src/tts.rs b/crates/sherpa-onnx-sys/src/tts.rs new file mode 100644 index 0000000000..6be479c56a --- /dev/null +++ b/crates/sherpa-onnx-sys/src/tts.rs @@ -0,0 +1,172 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float, c_void}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsVitsModelConfig { + pub model: *const c_char, + pub lexicon: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub noise_scale: c_float, + pub noise_scale_w: c_float, + pub length_scale: c_float, + pub dict_dir: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsMatchaModelConfig { + pub acoustic_model: *const c_char, + pub vocoder: *const c_char, + pub lexicon: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub noise_scale: c_float, + pub length_scale: c_float, + pub dict_dir: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsKokoroModelConfig { + pub model: *const c_char, + pub voices: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub length_scale: c_float, + pub dict_dir: *const c_char, + pub lexicon: *const c_char, + pub lang: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsKittenModelConfig { + pub model: *const c_char, + pub voices: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub length_scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsZipvoiceModelConfig { + pub tokens: *const c_char, + pub encoder: *const c_char, + pub decoder: *const c_char, + pub vocoder: *const c_char, + pub data_dir: *const c_char, + pub lexicon: *const c_char, + pub feat_scale: c_float, + pub t_shift: c_float, + pub target_rms: c_float, + pub guidance_scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsPocketModelConfig { + pub lm_flow: *const c_char, + pub lm_main: *const c_char, + pub encoder: *const c_char, + pub decoder: *const c_char, + pub text_conditioner: *const c_char, + pub vocab_json: *const c_char, + pub token_scores_json: *const c_char, + pub voice_embedding_cache_capacity: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsSupertonicModelConfig { + pub duration_predictor: *const c_char, + pub text_encoder: *const c_char, + pub vector_estimator: *const c_char, + pub vocoder: *const c_char, + pub tts_json: *const c_char, + pub unicode_indexer: *const c_char, + pub voice_style: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsModelConfig { + pub vits: OfflineTtsVitsModelConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, + pub matcha: OfflineTtsMatchaModelConfig, + pub kokoro: OfflineTtsKokoroModelConfig, + pub kitten: OfflineTtsKittenModelConfig, + pub zipvoice: OfflineTtsZipvoiceModelConfig, + pub pocket: OfflineTtsPocketModelConfig, + pub supertonic: OfflineTtsSupertonicModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsConfig { + pub model: OfflineTtsModelConfig, + pub rule_fsts: *const c_char, + pub max_num_sentences: i32, + pub rule_fars: *const c_char, + pub silence_scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SherpaOnnxGeneratedAudio { + pub samples: *const f32, + pub n: i32, + pub sample_rate: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SherpaOnnxGenerationConfig { + pub silence_scale: c_float, + pub speed: c_float, + pub sid: i32, + pub reference_audio: *const f32, + pub reference_audio_len: i32, + pub reference_sample_rate: i32, + pub reference_text: *const c_char, + pub num_steps: i32, + pub extra: *const c_char, +} + +pub type SherpaOnnxGeneratedAudioProgressCallbackWithArg = Option< + unsafe extern "C" fn(samples: *const f32, n: i32, progress: c_float, arg: *mut c_void) -> i32, +>; + +#[repr(C)] +pub struct SherpaOnnxOfflineTts { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineTts( + config: *const OfflineTtsConfig, + ) -> *const SherpaOnnxOfflineTts; + + pub fn SherpaOnnxDestroyOfflineTts(tts: *const SherpaOnnxOfflineTts); + + pub fn SherpaOnnxOfflineTtsSampleRate(tts: *const SherpaOnnxOfflineTts) -> i32; + + pub fn SherpaOnnxOfflineTtsNumSpeakers(tts: *const SherpaOnnxOfflineTts) -> i32; + + pub fn SherpaOnnxOfflineTtsGenerateWithConfig( + tts: *const SherpaOnnxOfflineTts, + text: *const c_char, + config: *const SherpaOnnxGenerationConfig, + callback: SherpaOnnxGeneratedAudioProgressCallbackWithArg, + arg: *mut c_void, + ) -> *const SherpaOnnxGeneratedAudio; + + pub fn SherpaOnnxDestroyOfflineTtsGeneratedAudio(p: *const SherpaOnnxGeneratedAudio); +} diff --git a/crates/sherpa-onnx-sys/src/vad.rs b/crates/sherpa-onnx-sys/src/vad.rs new file mode 100644 index 0000000000..b16b21deb9 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/vad.rs @@ -0,0 +1,85 @@ +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +pub struct SileroVadModelConfig { + pub model: *const c_char, + pub threshold: c_float, + pub min_silence_duration: c_float, + pub min_speech_duration: c_float, + pub window_size: i32, + pub max_speech_duration: c_float, +} + +#[repr(C)] +pub struct TenVadModelConfig { + pub model: *const c_char, + pub threshold: c_float, + pub min_silence_duration: c_float, + pub min_speech_duration: c_float, + pub window_size: i32, + pub max_speech_duration: c_float, +} + +#[repr(C)] +pub struct VadModelConfig { + pub silero_vad: SileroVadModelConfig, + pub sample_rate: i32, + pub num_threads: i32, + pub provider: *const c_char, + pub debug: i32, + pub ten_vad: TenVadModelConfig, +} + +#[repr(C)] +pub struct CircularBuffer { + _private: [u8; 0], +} + +#[repr(C)] +pub struct SpeechSegment { + pub start: i32, + pub samples: *mut f32, + pub n: i32, +} + +#[repr(C)] +pub struct VoiceActivityDetector { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateCircularBuffer(capacity: i32) -> *const CircularBuffer; + pub fn SherpaOnnxDestroyCircularBuffer(buffer: *const CircularBuffer); + pub fn SherpaOnnxCircularBufferPush(buffer: *const CircularBuffer, p: *const f32, n: i32); + pub fn SherpaOnnxCircularBufferGet( + buffer: *const CircularBuffer, + start_index: i32, + n: i32, + ) -> *const f32; + pub fn SherpaOnnxCircularBufferFree(p: *const f32); + pub fn SherpaOnnxCircularBufferPop(buffer: *const CircularBuffer, n: i32); + pub fn SherpaOnnxCircularBufferSize(buffer: *const CircularBuffer) -> i32; + pub fn SherpaOnnxCircularBufferHead(buffer: *const CircularBuffer) -> i32; + pub fn SherpaOnnxCircularBufferReset(buffer: *const CircularBuffer); + + pub fn SherpaOnnxCreateVoiceActivityDetector( + config: *const VadModelConfig, + buffer_size_in_seconds: c_float, + ) -> *const VoiceActivityDetector; + pub fn SherpaOnnxDestroyVoiceActivityDetector(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorAcceptWaveform( + p: *const VoiceActivityDetector, + samples: *const f32, + n: i32, + ); + pub fn SherpaOnnxVoiceActivityDetectorEmpty(p: *const VoiceActivityDetector) -> i32; + pub fn SherpaOnnxVoiceActivityDetectorDetected(p: *const VoiceActivityDetector) -> i32; + pub fn SherpaOnnxVoiceActivityDetectorPop(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorClear(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorFront( + p: *const VoiceActivityDetector, + ) -> *const SpeechSegment; + pub fn SherpaOnnxDestroySpeechSegment(p: *const SpeechSegment); + pub fn SherpaOnnxVoiceActivityDetectorReset(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorFlush(p: *const VoiceActivityDetector); +} diff --git a/crates/sherpa-onnx-sys/src/wave.rs b/crates/sherpa-onnx-sys/src/wave.rs new file mode 100644 index 0000000000..c3c8f860fe --- /dev/null +++ b/crates/sherpa-onnx-sys/src/wave.rs @@ -0,0 +1,30 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SherpaOnnxWave { + /// Samples normalized to [-1, 1] + pub samples: *const f32, + pub sample_rate: i32, + pub num_samples: i32, +} + +extern "C" { + /// Read a WAV file. Returns NULL on error. + pub fn SherpaOnnxReadWave(filename: *const c_char) -> *const SherpaOnnxWave; + + /// Free memory allocated by SherpaOnnxReadWave + pub fn SherpaOnnxFreeWave(wave: *const SherpaOnnxWave); + + /// Write a WAV file. Returns 1 on success, 0 on failure. + pub fn SherpaOnnxWriteWave( + samples: *const f32, + n: i32, + sample_rate: i32, + filename: *const c_char, + ) -> i32; +} From cb7b330fe3c07eea30f01a5d13b6be90ffaa2847 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 28 Jul 2026 06:00:32 -0400 Subject: [PATCH 5/8] docs(voice): clarify callback and vendoring Signed-off-by: John Tennant --- crates/buzz-voice/src/pocket.rs | 4 ---- crates/sherpa-onnx-sys/README.md | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/buzz-voice/src/pocket.rs b/crates/buzz-voice/src/pocket.rs index 277fecd3d6..db4f7b8f5e 100644 --- a/crates/buzz-voice/src/pocket.rs +++ b/crates/buzz-voice/src/pocket.rs @@ -425,10 +425,6 @@ impl PocketTts { ..Default::default() }; - // No progress callback — synthesis is fast enough that returning the - // whole buffer at once keeps the lookahead pipelining in `tts.rs` - // simple. `None:: bool>` pins the callback type for the - // `generate_with_config` generic parameter. let audio = self .inner .generate_with_config(&prepared.text, &cfg, callback) diff --git a/crates/sherpa-onnx-sys/README.md b/crates/sherpa-onnx-sys/README.md index 06db754497..689fa55575 100644 --- a/crates/sherpa-onnx-sys/README.md +++ b/crates/sherpa-onnx-sys/README.md @@ -4,8 +4,8 @@ -Buzz carries this source-identical crates.io package so its build script can -link the official merged iOS archives and Android shared libraries through +Buzz carries the crates.io 1.13.4 FFI sources and locally patches the build +script to link caller-supplied official iOS and Android libraries through `SHERPA_ONNX_LIB_DIR`. Desktop archive selection remains unchanged. ### Supported functions From d75e9977985468596ebf77159b3bc43b04de5a60 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 28 Jul 2026 06:19:22 -0400 Subject: [PATCH 6/8] fix(voice): label reusable diagnostics Signed-off-by: John Tennant --- crates/buzz-voice/src/pocket.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/buzz-voice/src/pocket.rs b/crates/buzz-voice/src/pocket.rs index db4f7b8f5e..c070918d4d 100644 --- a/crates/buzz-voice/src/pocket.rs +++ b/crates/buzz-voice/src/pocket.rs @@ -438,7 +438,7 @@ impl PocketTts { let sample_rate = audio.sample_rate(); if sample_rate != SAMPLE_RATE as i32 { eprintln!( - "buzz-desktop: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ + "buzz-voice: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ (expected {SAMPLE_RATE}Hz); playback speed may be wrong" ); } From 315defeeb23e85b4fa1f803807e206f411152e0a Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 28 Jul 2026 06:33:52 -0400 Subject: [PATCH 7/8] fix(ci): preserve patched sherpa version in cargo chef Signed-off-by: John Tennant --- Cargo.toml | 3 +-- Dockerfile | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4761ef4f95..f7bbad294b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,10 +27,9 @@ members = [ "crates/buzz-relay-mesh", "crates/buzz-dev-mcp", "crates/buzz-voice", - "crates/sherpa-onnx-sys", "examples/countdown-bot", ] -exclude = ["desktop/src-tauri"] +exclude = ["desktop/src-tauri", "crates/sherpa-onnx-sys"] resolver = "2" [workspace.package] diff --git a/Dockerfile b/Dockerfile index d883ac6b01..6699273784 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update \ # locations. The normal runtime strips it below; runtime-debug retains it. ENV CARGO_PROFILE_RELEASE_DEBUG=line-tables-only COPY --from=planner /build/recipe.json recipe.json +# The sherpa sys patch keeps its published 1.13.4 version so Cargo can replace +# the registry crate. It is excluded from the workspace recipe because +# cargo-chef masks workspace package versions to 0.0.1. +COPY --from=planner /build/crates/sherpa-onnx-sys crates/sherpa-onnx-sys # Cook the full workspace recipe — relay deps include workspace siblings, so # scoping to -p buzz-relay misses transitive deps and re-builds them later. RUN cargo chef cook --release --recipe-path recipe.json From 0f97b7ef6d2f9aa87dea9a639223c39d9f0f9066 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 28 Jul 2026 06:39:19 -0400 Subject: [PATCH 8/8] fix(ci): provide sherpa patch to push gateway cook Signed-off-by: John Tennant --- Dockerfile.push-gateway | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile.push-gateway b/Dockerfile.push-gateway index 202262d222..98204ff0d4 100644 --- a/Dockerfile.push-gateway +++ b/Dockerfile.push-gateway @@ -15,6 +15,9 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential pkg-config libssl-dev ca-certificates \ && rm -rf /var/lib/apt/lists/* COPY --from=planner /build/recipe.json recipe.json +# Keep the patched sherpa sys crate at its published version; cargo-chef masks +# workspace package versions, so the patch remains outside the workspace recipe. +COPY --from=planner /build/crates/sherpa-onnx-sys crates/sherpa-onnx-sys RUN cargo chef cook --release --recipe-path recipe.json COPY . . RUN cargo build --release --locked -p buzz-push-gateway --bin buzz-push-gateway \