diff --git a/.gitignore b/.gitignore index 65ddcaf1c4..3ef0226174 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /target/ /dist/ /admin-web/dist/ +/mobile/.generated/voice/ # lefthook-generated hook scripts (machine-specific) .hooks/ diff --git a/Cargo.lock b/Cargo.lock index 3b60dc4579..5faa667905 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,6 +43,20 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if 1.0.4", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -131,7 +145,7 @@ checksum = "5d0a66767aaf7d483c556386fb68ca2fba9347684d8bb17a4bd8b755851870f7" dependencies = [ "arrayvec", "aws-lc-rs", - "base64", + "base64 0.22.1", "byteorder", "minicbor", "rustls-pki-types", @@ -402,7 +416,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" dependencies = [ - "base64", + "base64 0.22.1", "http", "log", "rustls", @@ -473,7 +487,7 @@ checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "axum-macros", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", @@ -549,6 +563,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -567,6 +587,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bip39" version = "2.2.2" @@ -765,7 +791,7 @@ name = "buzz-acp" version = "0.1.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "buzz-core", "buzz-persona", "buzz-sdk", @@ -825,7 +851,7 @@ dependencies = [ "arc-swap", "async-trait", "axum", - "base64", + "base64 0.22.1", "getrandom 0.4.3", "hex", "nix 0.31.3", @@ -884,7 +910,7 @@ name = "buzz-cli" version = "0.1.0" dependencies = [ "axum", - "base64", + "base64 0.22.1", "buzz-core", "buzz-persona", "buzz-sdk", @@ -925,7 +951,7 @@ dependencies = [ name = "buzz-core" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "hex", "hmac 0.13.0", @@ -965,7 +991,7 @@ dependencies = [ name = "buzz-dev-mcp" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "buzz-cli", "buzz-core", "git-credential-nostr", @@ -1092,7 +1118,7 @@ dependencies = [ "appattest", "async-trait", "axum", - "base64", + "base64 0.22.1", "byteorder", "chrono", "getrandom 0.4.3", @@ -1127,7 +1153,7 @@ dependencies = [ "async-compression", "async-trait", "axum", - "base64", + "base64 0.22.1", "buzz-audit", "buzz-auth", "buzz-conformance", @@ -1237,7 +1263,7 @@ name = "buzz-test-client" version = "0.1.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "buzz-core", "buzz-sdk", "buzz-ws-client", @@ -1262,6 +1288,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "ort", + "ort-sys", + "rand 0.10.1", + "regex", + "sentencepiece-model", + "serde", + "serde_json", + "sherpa-onnx", + "tokenizers", +] + +[[package]] +name = "buzz-voice-mobile" +version = "0.1.0" +dependencies = [ + "buzz-voice", + "serde_json", +] + [[package]] name = "buzz-workflow" version = "0.1.0" @@ -1329,6 +1378,26 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "castaway" version = "0.2.4" @@ -1577,6 +1646,7 @@ dependencies = [ "itoa", "rustversion", "ryu", + "serde", "static_assertions", ] @@ -2137,6 +2207,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -2174,7 +2253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -2626,6 +2705,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "etcetera" version = "0.11.0" @@ -2693,6 +2778,17 @@ dependencies = [ "regex", ] +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fast-srgb8" version = "1.0.0" @@ -3084,7 +3180,7 @@ dependencies = [ name = "git-credential-nostr" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "nostr", "serde_json", "zeroize", @@ -3094,7 +3190,7 @@ dependencies = [ name = "git-sign-nostr" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "hex", "libc", @@ -3269,7 +3365,7 @@ version = "1.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f89305dc8fe34e165eaf0eb12b6e294e12381d9df9a431bcc52a5809bab4319" dependencies = [ - "base64", + "base64 0.22.1", "bon", "bytes", "futures", @@ -3575,7 +3671,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -4359,6 +4455,39 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "logos" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "syn 2.0.117", +] + +[[package]] +name = "logos-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" +dependencies = [ + "logos-codegen", +] + [[package]] name = "loom" version = "0.7.2" @@ -4418,6 +4547,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + [[package]] name = "matchers" version = "0.2.0" @@ -4433,6 +4578,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-async" version = "0.2.11" @@ -4529,7 +4684,7 @@ source = "git+https://github.com/Mesh-LLM/mesh-llm.git?tag=v0.73.1#43103c5c40292 dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "crypto_box", "ed25519-dalek", @@ -4542,7 +4697,7 @@ dependencies = [ "mesh-llm-types", "model-artifact", "nostr-sdk", - "prost", + "prost 0.14.3", "rand 0.10.1", "rustls", "serde", @@ -4631,7 +4786,7 @@ dependencies = [ "argon2", "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "chacha20poly1305", "chrono", @@ -4681,7 +4836,7 @@ dependencies = [ "opentelemetry 0.31.0", "opentelemetry-otlp 0.31.1", "opentelemetry_sdk 0.31.0", - "prost", + "prost 0.14.3", "rand 0.10.1", "regex-lite", "reqwest 0.12.28", @@ -4721,7 +4876,7 @@ version = "0.73.1" source = "git+https://github.com/Mesh-LLM/mesh-llm.git?tag=v0.73.1#43103c5c40292be688ac0261129bcbab0e7b9132" dependencies = [ "argon2", - "base64", + "base64 0.22.1", "chacha20poly1305", "chrono", "crypto_box", @@ -4769,8 +4924,8 @@ source = "git+https://github.com/Mesh-LLM/mesh-llm.git?tag=v0.73.1#43103c5c40292 dependencies = [ "anyhow", "async-trait", - "prost", - "prost-build", + "prost 0.14.3", + "prost-build 0.14.3", "protoc-bin-vendored", "rmcp", "schemars", @@ -4805,7 +4960,7 @@ dependencies = [ "anyhow", "hex", "iroh", - "prost", + "prost 0.14.3", "serde_json", "sha2 0.10.9", ] @@ -4936,7 +5091,7 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" dependencies = [ - "base64", + "base64 0.22.1", "evmap", "http-body-util", "hyper", @@ -4974,6 +5129,28 @@ dependencies = [ "sketches-ddsketch", ] +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if 1.0.4", + "miette-derive", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "mime" version = "0.3.17" @@ -5119,6 +5296,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "more-asserts" version = "0.3.1" @@ -5225,6 +5424,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk-context" version = "0.1.1" @@ -5461,7 +5675,7 @@ version = "0.44.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e826dd648489de2c5b293920e20b92932ef820302007c1987c758d4d06eeb2cf" dependencies = [ - "base64", + "base64 0.22.1", "bech32", "bip39", "bitcoin_hashes", @@ -5932,7 +6146,7 @@ dependencies = [ "opentelemetry-http", "opentelemetry-proto 0.31.0", "opentelemetry_sdk 0.31.0", - "prost", + "prost 0.14.3", "reqwest 0.12.28", "thiserror 2.0.18", ] @@ -5947,7 +6161,7 @@ dependencies = [ "opentelemetry 0.32.0", "opentelemetry-proto 0.32.0", "opentelemetry_sdk 0.32.1", - "prost", + "prost 0.14.3", "thiserror 2.0.18", "tokio", "tonic", @@ -5960,11 +6174,11 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ - "base64", + "base64 0.22.1", "const-hex", "opentelemetry 0.31.0", "opentelemetry_sdk 0.31.0", - "prost", + "prost 0.14.3", "serde", "serde_json", "tonic", @@ -5979,7 +6193,7 @@ checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry 0.32.0", "opentelemetry_sdk 0.32.1", - "prost", + "prost 0.14.3", "tonic", "tonic-prost", ] @@ -6061,6 +6275,24 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" + [[package]] name = "os_str_bytes" version = "6.6.1" @@ -6242,6 +6474,16 @@ dependencies = [ "pest", ] +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", + "indexmap", +] + [[package]] name = "petgraph" version = "0.8.3" @@ -6374,7 +6616,7 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ - "base64", + "base64 0.22.1", "indexmap", "quick-xml 0.39.4", "serde", @@ -6440,13 +6682,22 @@ dependencies = [ "serde", ] +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "portmapper" version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb3713e4977408279158444a18c1a01ac9bf2e7eaf1fbfd1a19ac9cd18d90721" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "derive_more", "hyper-util", @@ -6616,6 +6867,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.13.5", +] + [[package]] name = "prost" version = "0.14.3" @@ -6623,7 +6884,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.14.3", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph 0.7.1", + "prettyplease", + "prost 0.13.5", + "prost-types 0.13.5", + "regex", + "syn 2.0.117", + "tempfile", ] [[package]] @@ -6636,15 +6917,28 @@ dependencies = [ "itertools", "log", "multimap", - "petgraph", + "petgraph 0.8.3", "prettyplease", - "prost", - "prost-types", + "prost 0.14.3", + "prost-types 0.14.3", "regex", "syn 2.0.117", "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "prost-derive" version = "0.14.3" @@ -6658,13 +6952,35 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-reflect" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5edd582b62f5cde844716e66d92565d7faf7ab1445c8cebce6e00fba83ddb2" +dependencies = [ + "logos", + "miette", + "once_cell", + "prost 0.13.5", + "prost-types 0.13.5", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost 0.13.5", +] + [[package]] name = "prost-types" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ - "prost", + "prost 0.14.3", ] [[package]] @@ -6731,6 +7047,33 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" +[[package]] +name = "protox" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f352af331bf637b8ecc720f7c87bf903d2571fa2e14a66e9b2558846864b54a" +dependencies = [ + "bytes", + "miette", + "prost 0.13.5", + "prost-reflect", + "prost-types 0.13.5", + "protox-parse", + "thiserror 1.0.69", +] + +[[package]] +name = "protox-parse" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a462d115462c080ae000c29a47f0b3985737e5d3a995fcdbcaa5c782068dde" +dependencies = [ + "logos", + "miette", + "prost-types 0.13.5", + "thiserror 1.0.69", +] + [[package]] name = "pulldown-cmark" version = "0.13.4" @@ -7036,7 +7379,7 @@ dependencies = [ "thiserror 2.0.18", "unicode-segmentation", "unicode-truncate", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -7099,7 +7442,7 @@ dependencies = [ "strum", "time", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -7111,6 +7454,43 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redb" version = "3.1.3" @@ -7232,7 +7612,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-channel", @@ -7280,7 +7660,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -7370,7 +7750,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0810a9f717d9828f475fe1f629f4c305c8464b7f496c3a854b58d29e65f4058e" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "futures", @@ -7438,7 +7818,7 @@ dependencies = [ "async-trait", "aws-creds", "aws-region", - "base64", + "base64 0.22.1", "bytes", "cfg-if 1.0.4", "futures-util", @@ -7839,6 +8219,18 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" +[[package]] +name = "sentencepiece-model" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b87bf750a8322c3236d7aa63c1f4a6862187d00d2d8b038e1dfe263bfe43ec" +dependencies = [ + "miette", + "prost 0.13.5", + "prost-build 0.13.5", + "protox", +] + [[package]] name = "serde" version = "1.0.228" @@ -8049,6 +8441,26 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "sherpa-onnx" +version = "1.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b142d3f255cb4e4b7808ea25869db6f5714e0a3550da355234483b4db552055" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.13.4" +dependencies = [ + "bzip2", + "tar", + "ureq", +] + [[package]] name = "shlex" version = "1.3.0" @@ -8184,8 +8596,8 @@ name = "skippy-protocol" version = "0.73.1" source = "git+https://github.com/Mesh-LLM/mesh-llm.git?tag=v0.73.1#43103c5c40292be688ac0261129bcbab0e7b9132" dependencies = [ - "prost", - "prost-build", + "prost 0.14.3", + "prost-build 0.14.3", "protoc-bin-vendored", "serde", ] @@ -8212,7 +8624,7 @@ dependencies = [ "anyhow", "async-trait", "axum", - "base64", + "base64 0.22.1", "blake3", "clap", "futures-util", @@ -8319,6 +8731,18 @@ dependencies = [ "der", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "sprig" version = "0.1.0" @@ -8347,7 +8771,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "cfg-if 1.0.4", "chrono", @@ -8453,7 +8877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags 2.13.0", "byteorder", "chrono", @@ -8691,7 +9115,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce91f2f0ec87dff7e6bcbbeb267439aa1188703003c6055193c821487400432" dependencies = [ - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -8765,9 +9189,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bitflags 2.13.0", - "fancy-regex", + "fancy-regex 0.11.0", "filedescriptor", "finl_unicode", "fixedbitset 0.4.2", @@ -8917,6 +9341,39 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str 0.9.1", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex 0.14.0", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "paste", + "rand 0.9.4", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.52.3" @@ -9052,7 +9509,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dad543404f98bfc969aeb71994105c592acfc6c43323fddcd016bb208d1c65cb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-sink", @@ -9153,7 +9610,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http", @@ -9182,7 +9639,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", - "prost", + "prost 0.14.3", "tonic", ] @@ -9192,8 +9649,8 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" dependencies = [ - "prost", - "prost-types", + "prost 0.14.3", + "prost-types 0.14.3", "tonic", ] @@ -9482,6 +9939,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" @@ -9502,9 +9968,15 @@ checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5" dependencies = [ "itertools", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.2" @@ -9517,6 +9989,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "universal-hash" version = "0.5.1" @@ -9539,6 +10017,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.8" @@ -10530,7 +11024,7 @@ checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "clap", "crc32fast", @@ -10567,7 +11061,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "blake3", "bytemuck", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 3ac7ee4cce..c916e09040 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,9 +26,11 @@ members = [ "crates/buzz-pair-relay", "crates/buzz-relay-mesh", "crates/buzz-dev-mcp", + "crates/buzz-voice", + "crates/buzz-voice-mobile", "examples/countdown-bot", ] -exclude = ["desktop/src-tauri"] +exclude = ["desktop/src-tauri", "crates/sherpa-onnx-sys"] resolver = "2" [workspace.package] @@ -170,3 +172,4 @@ strip = true # allowlist for the auth token). Revert to crates.io once #449 lands upstream. [patch.crates-io] aws-creds = { git = "https://github.com/tlongwell-block/rust-s3", rev = "c9fce3620dd434c1f810101d672cf384268dbb0f" } +sherpa-onnx-sys = { path = "crates/sherpa-onnx-sys" } diff --git a/Dockerfile b/Dockerfile index d883ac6b01..6699273784 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update \ # locations. The normal runtime strips it below; runtime-debug retains it. ENV CARGO_PROFILE_RELEASE_DEBUG=line-tables-only COPY --from=planner /build/recipe.json recipe.json +# The sherpa sys patch keeps its published 1.13.4 version so Cargo can replace +# the registry crate. It is excluded from the workspace recipe because +# cargo-chef masks workspace package versions to 0.0.1. +COPY --from=planner /build/crates/sherpa-onnx-sys crates/sherpa-onnx-sys # Cook the full workspace recipe — relay deps include workspace siblings, so # scoping to -p buzz-relay misses transitive deps and re-builds them later. RUN cargo chef cook --release --recipe-path recipe.json diff --git a/Dockerfile.push-gateway b/Dockerfile.push-gateway index 202262d222..98204ff0d4 100644 --- a/Dockerfile.push-gateway +++ b/Dockerfile.push-gateway @@ -15,6 +15,9 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential pkg-config libssl-dev ca-certificates \ && rm -rf /var/lib/apt/lists/* COPY --from=planner /build/recipe.json recipe.json +# Keep the patched sherpa sys crate at its published version; cargo-chef masks +# workspace package versions, so the patch remains outside the workspace recipe. +COPY --from=planner /build/crates/sherpa-onnx-sys crates/sherpa-onnx-sys RUN cargo chef cook --release --recipe-path recipe.json COPY . . RUN cargo build --release --locked -p buzz-push-gateway --bin buzz-push-gateway \ diff --git a/Justfile b/Justfile index bcef8983bc..e5601607ba 100644 --- a/Justfile +++ b/Justfile @@ -276,6 +276,7 @@ test-unit: #!/usr/bin/env bash if command -v cargo-nextest &>/dev/null; then cargo nextest run -p buzz-core -p buzz-auth --lib + cargo nextest run -p buzz-voice --lib # buzz-db migrator/lint tests: pure SQL-parsing unit tests (no infra). # They guard the embedded-migrator invariant (exactly the consolidated # 0001; cutover/backfill stays an operator script, not startup state) diff --git a/crates/buzz-voice-mobile/Cargo.toml b/crates/buzz-voice-mobile/Cargo.toml new file mode 100644 index 0000000000..bd3ba760f1 --- /dev/null +++ b/crates/buzz-voice-mobile/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "buzz-voice-mobile" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Minimal mobile C ABI for Buzz Pocket TTS" + +[lib] +crate-type = ["rlib", "staticlib", "cdylib"] + +[features] +default = ["static"] +static = ["buzz-voice/static"] +shared = ["buzz-voice/shared"] + +[dependencies] +buzz-voice = { path = "../buzz-voice", default-features = false } +serde_json = { workspace = true } diff --git a/crates/buzz-voice-mobile/include/buzz_voice_mobile.h b/crates/buzz-voice-mobile/include/buzz_voice_mobile.h new file mode 100644 index 0000000000..a82fac4d21 --- /dev/null +++ b/crates/buzz-voice-mobile/include/buzz_voice_mobile.h @@ -0,0 +1,32 @@ +#ifndef BUZZ_VOICE_MOBILE_H +#define BUZZ_VOICE_MOBILE_H + +#include +#include + +typedef struct BuzzVoiceEngineResult { + void *engine; + char *error; +} BuzzVoiceEngineResult; + +typedef struct BuzzVoicePcm { + int16_t *samples; + size_t len; + uint32_t sample_rate; + char *error; +} BuzzVoicePcm; + +#define BUZZ_VOICE_PRECISION_FP32 0 +#define BUZZ_VOICE_PRECISION_INT8 1 + +BuzzVoiceEngineResult buzz_voice_engine_create(const char *model_dir, + uint8_t precision); +char *buzz_voice_prepare_chunks_json(void *engine, const char *text); +BuzzVoicePcm buzz_voice_engine_synthesize(void *engine, const char *text); +void buzz_voice_engine_cancel(void *engine); +void buzz_voice_engine_reset_cancel(void *engine); +void buzz_voice_engine_destroy(void *engine); +void buzz_voice_pcm_free(BuzzVoicePcm pcm); +void buzz_voice_string_free(char *value); + +#endif diff --git a/crates/buzz-voice-mobile/src/lib.rs b/crates/buzz-voice-mobile/src/lib.rs new file mode 100644 index 0000000000..ca6339ad28 --- /dev/null +++ b/crates/buzz-voice-mobile/src/lib.rs @@ -0,0 +1,342 @@ +//! Minimal C ABI for running [`buzz_voice::pocket`] on mobile. + +#![deny(unsafe_op_in_unsafe_fn)] + +use std::ffi::{c_char, c_void, CStr, CString}; +use std::ptr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use buzz_voice::pocket::{ + load_text_to_speech_with_options, load_voice_style, PocketLoadOptions, PocketModelSelection, + PocketPrecision, PocketTts, VoiceStyle, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT, +}; +use buzz_voice::preparation::{prepare_tts_chunks, shape_tts_chunk}; + +struct Engine { + tts: PocketTts, + voice: VoiceStyle, + cancelled: Arc, +} + +/// Result returned by [`buzz_voice_engine_create`]. +#[repr(C)] +pub struct BuzzVoiceEngineResult { + /// Opaque engine handle, or null when creation failed. + pub engine: *mut c_void, + /// Owned UTF-8 error string, or null on success. + pub error: *mut c_char, +} + +/// Owned 24 kHz mono signed-16-bit PCM returned by +/// [`buzz_voice_engine_synthesize`]. +#[repr(C)] +pub struct BuzzVoicePcm { + /// Owned sample buffer, or null when synthesis failed. + pub samples: *mut i16, + /// Number of samples in `samples`. + pub len: usize, + /// Sample rate in Hz. + pub sample_rate: u32, + /// Owned UTF-8 error string, or null on success. + pub error: *mut c_char, +} + +fn owned_error(message: impl Into) -> *mut c_char { + let message = message.into().replace('\0', " "); + CString::new(message) + .map(CString::into_raw) + .unwrap_or(ptr::null_mut()) +} + +fn engine_error(message: impl Into) -> BuzzVoiceEngineResult { + BuzzVoiceEngineResult { + engine: ptr::null_mut(), + error: owned_error(message), + } +} + +fn pcm_error(message: impl Into) -> BuzzVoicePcm { + BuzzVoicePcm { + samples: ptr::null_mut(), + len: 0, + sample_rate: SAMPLE_RATE, + error: owned_error(message), + } +} + +fn input_string(value: *const c_char, label: &str) -> Result { + if value.is_null() { + return Err(format!("{label} is null")); + } + // SAFETY: The C ABI requires a non-null, NUL-terminated string that stays + // alive for the duration of this call. We copy it before returning. + let value = unsafe { CStr::from_ptr(value) }; + value + .to_str() + .map(str::to_owned) + .map_err(|error| format!("{label} is not UTF-8: {error}")) +} + +/// Create and retain one Pocket engine and its reference voice. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_create( + model_dir: *const c_char, + precision: u8, +) -> BuzzVoiceEngineResult { + let model_dir = match input_string(model_dir, "model_dir") { + Ok(value) => value, + Err(error) => return engine_error(error), + }; + let precision = match precision { + value if value == PocketPrecision::Fp32 as u8 => PocketPrecision::Fp32, + value if value == PocketPrecision::Int8 as u8 => PocketPrecision::Int8, + value => return engine_error(format!("unsupported Pocket precision: {value}")), + }; + let voice_path = std::path::Path::new(&model_dir) + .join(DEFAULT_VOICE) + .with_extension(VOICE_FILE_EXT); + let options = PocketLoadOptions { + model: PocketModelSelection::English2026_04(precision), + ..PocketLoadOptions::default() + }; + let tts = match load_text_to_speech_with_options(&model_dir, options) { + Ok(value) => value, + Err(error) => return engine_error(error), + }; + let voice = match load_voice_style(&voice_path) { + Ok(value) => value, + Err(error) => return engine_error(error), + }; + // Pay the model's first-generation setup cost during engine creation so + // the first submitted assistant sentence is not the warm-up utterance. + if let Err(error) = tts.synth_chunk("warmup", "en", &voice, 1) { + eprintln!("buzz-voice-mobile: Pocket warm-up failed: {error}"); + } + let engine = Box::new(Engine { + tts, + voice, + cancelled: Arc::new(AtomicBool::new(false)), + }); + BuzzVoiceEngineResult { + engine: Box::into_raw(engine).cast(), + error: ptr::null_mut(), + } +} + +/// Synthesize one utterance into owned signed-16-bit PCM. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_synthesize( + engine: *mut c_void, + text: *const c_char, +) -> BuzzVoicePcm { + if engine.is_null() { + return pcm_error("engine is null"); + } + let text = match input_string(text, "text") { + Ok(value) => value, + Err(error) => return pcm_error(error), + }; + // SAFETY: A handle returned by `buzz_voice_engine_create` remains owned by + // the caller until `buzz_voice_engine_destroy`; synthesis and destroy must + // not overlap. Cancellation only touches the atomic flag. + let engine = unsafe { &*(engine.cast::()) }; + let cancelled = Arc::clone(&engine.cancelled); + let samples = match engine.tts.synth_chunk_with_callback( + &text, + "en", + &engine.voice, + 1, + Some(move |_: &[f32], _| !cancelled.load(Ordering::Acquire)), + ) { + Ok(value) => value, + Err(error) => return pcm_error(error), + }; + if engine.cancelled.load(Ordering::Acquire) { + return pcm_error("synthesis cancelled"); + } + + let samples: Box<[i16]> = shape_tts_chunk(samples) + .into_iter() + .map(|sample| { + let sample = if sample.is_finite() { sample } else { 0.0 }; + (sample.clamp(-1.0, 1.0) * i16::MAX as f32).round() as i16 + }) + .collect::>() + .into_boxed_slice(); + let len = samples.len(); + BuzzVoicePcm { + samples: Box::into_raw(samples).cast::(), + len, + sample_rate: SAMPLE_RATE, + error: ptr::null_mut(), + } +} + +/// Prepare submitted assistant text and return an owned JSON string array. +/// +/// The returned pointer is released with [`buzz_voice_string_free`]. A null +/// pointer indicates invalid input or an unexpected serialization failure. +#[no_mangle] +pub extern "C" fn buzz_voice_prepare_chunks_json( + engine: *mut c_void, + text: *const c_char, +) -> *mut c_char { + if engine.is_null() { + return ptr::null_mut(); + } + let text = match input_string(text, "text") { + Ok(value) => value, + Err(_) => return ptr::null_mut(), + }; + // SAFETY: See `buzz_voice_engine_synthesize`. + let engine = unsafe { &*(engine.cast::()) }; + let mut chunks = Vec::new(); + for prepared in prepare_tts_chunks(&text) { + let model_chunks = match engine.tts.split_text_into_chunks(&prepared) { + Ok(value) => value, + Err(_) => return ptr::null_mut(), + }; + chunks.extend(model_chunks); + } + let json = match serde_json::to_string(&chunks) { + Ok(value) => value, + Err(_) => return ptr::null_mut(), + }; + CString::new(json) + .map(CString::into_raw) + .unwrap_or(ptr::null_mut()) +} + +/// Request cancellation of the current and subsequent synthesis calls. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_cancel(engine: *mut c_void) { + if engine.is_null() { + return; + } + // SAFETY: See `buzz_voice_engine_synthesize`. The engine remains alive + // while this atomic cancellation request is issued. + let engine = unsafe { &*(engine.cast::()) }; + engine.cancelled.store(true, Ordering::Release); +} + +/// Clear sticky cancellation before beginning a new playback request. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_reset_cancel(engine: *mut c_void) { + if engine.is_null() { + return; + } + // SAFETY: See `buzz_voice_engine_synthesize`. + let engine = unsafe { &*(engine.cast::()) }; + engine.cancelled.store(false, Ordering::Release); +} + +/// Destroy an engine after any synthesis call has returned. +#[no_mangle] +pub extern "C" fn buzz_voice_engine_destroy(engine: *mut c_void) { + if engine.is_null() { + return; + } + // SAFETY: The pointer came from `Box::into_raw` in create and is consumed + // exactly once by this function. + unsafe { drop(Box::from_raw(engine.cast::())) }; +} + +/// Release a PCM value returned by [`buzz_voice_engine_synthesize`]. +#[no_mangle] +pub extern "C" fn buzz_voice_pcm_free(pcm: BuzzVoicePcm) { + if !pcm.samples.is_null() { + // SAFETY: The allocation came from `Box<[i16]>` in synthesis and is + // consumed exactly once here with the same slice length. + let slice = ptr::slice_from_raw_parts_mut(pcm.samples, pcm.len); + unsafe { drop(Box::from_raw(slice)) }; + } + buzz_voice_string_free(pcm.error); +} + +/// Release an error string returned by this ABI. +#[no_mangle] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub extern "C" fn buzz_voice_string_free(value: *mut c_char) { + if value.is_null() { + return; + } + // SAFETY: Error pointers are produced by `CString::into_raw` and consumed + // exactly once here. + unsafe { drop(CString::from_raw(value)) }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn null_inputs_return_owned_errors() { + let engine = buzz_voice_engine_create(ptr::null(), PocketPrecision::Fp32 as u8); + assert!(engine.engine.is_null()); + assert!(!engine.error.is_null()); + buzz_voice_string_free(engine.error); + + let pcm = buzz_voice_engine_synthesize(ptr::null_mut(), ptr::null()); + assert!(pcm.samples.is_null()); + assert_eq!(pcm.len, 0); + assert_eq!(pcm.sample_rate, SAMPLE_RATE); + assert!(!pcm.error.is_null()); + buzz_voice_pcm_free(pcm); + } + + #[test] + fn chunk_preparation_requires_an_engine() { + let input = CString::new("First. Second. Third.").expect("test input"); + let json = buzz_voice_prepare_chunks_json(ptr::null_mut(), input.as_ptr()); + assert!(json.is_null()); + } + + #[test] + fn invalid_precision_returns_an_owned_error() { + let model = CString::new("/tmp/pocket").expect("test path"); + let engine = buzz_voice_engine_create(model.as_ptr(), u8::MAX); + assert!(engine.engine.is_null()); + assert!(!engine.error.is_null()); + buzz_voice_string_free(engine.error); + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"] + fn april_fp32_abi_prepares_and_synthesizes_valid_pcm() { + let model = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to an April FP32 bundle"); + let model = CString::new(model).expect("test model path"); + let result = buzz_voice_engine_create(model.as_ptr(), PocketPrecision::Fp32 as u8); + if !result.error.is_null() { + // SAFETY: The ABI returned a live NUL-terminated owned error. + let error = unsafe { CStr::from_ptr(result.error) } + .to_string_lossy() + .into_owned(); + buzz_voice_string_free(result.error); + panic!("engine creation failed: {error}"); + } + assert!(!result.engine.is_null()); + + let text = CString::new( + "Pocket voice is running on mobile. This second sentence verifies shared chunking.", + ) + .expect("test text"); + let json = buzz_voice_prepare_chunks_json(result.engine, text.as_ptr()); + assert!(!json.is_null()); + // SAFETY: The ABI returned a live NUL-terminated owned JSON string. + let chunks: Vec = + serde_json::from_slice(unsafe { CStr::from_ptr(json) }.to_bytes()).expect("chunk JSON"); + buzz_voice_string_free(json); + assert_eq!(chunks.len(), 2); + + let prompt = CString::new(chunks[0].as_str()).expect("prepared prompt"); + let pcm = buzz_voice_engine_synthesize(result.engine, prompt.as_ptr()); + assert!(pcm.error.is_null()); + assert!(!pcm.samples.is_null()); + assert!(pcm.len > 24_000 / 4); + assert_eq!(pcm.sample_rate, 24_000); + buzz_voice_pcm_free(pcm); + buzz_voice_engine_destroy(result.engine); + } +} diff --git a/crates/buzz-voice/Cargo.toml b/crates/buzz-voice/Cargo.toml new file mode 100644 index 0000000000..e492a9e7f1 --- /dev/null +++ b/crates/buzz-voice/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "buzz-voice" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Reusable local voice primitives for Buzz" + +[features] +default = ["static"] +static = ["sherpa-onnx/static"] +shared = ["sherpa-onnx/shared"] + +[dependencies] +ort = { version = "=2.0.0-rc.12", default-features = false, features = ["api-24", "ndarray", "std"] } +ort-sys = { version = "=2.0.0-rc.12", features = ["disable-linking"] } +rand = { workspace = true } +regex = "1" +sentencepiece-model = "0.1" +serde = { workspace = true } +serde_json = { workspace = true } +sherpa-onnx = { version = "1.12", default-features = false } +tokenizers = { version = "0.22", default-features = false, features = ["fancy-regex"] } diff --git a/crates/buzz-voice/examples/pocket_april_onset.rs b/crates/buzz-voice/examples/pocket_april_onset.rs new file mode 100644 index 0000000000..b8ba7bea75 --- /dev/null +++ b/crates/buzz-voice/examples/pocket_april_onset.rs @@ -0,0 +1,86 @@ +//! Generate one raw, untrimmed April Pocket TTS onset-stress clip. +//! +//! Each sentence is a separate synthesis call, making every sentence start an +//! explicit model-generation boundary. Outputs are concatenated without fades, +//! trimming, normalization, or gain; only 250 ms of digital silence is inserted +//! between calls so boundaries are easy to identify while listening. +//! +//! Usage: +//! cargo run --release -p buzz-voice --example pocket_april_onset -- \ +//! [fp32|int8] + +use std::path::PathBuf; + +use buzz_voice::pocket::{ + load_text_to_speech_with_options, load_voice_style, PocketLoadOptions, PocketModelSelection, + PocketPrecision, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT, +}; + +const PASSAGE: &[&str] = &[ + "Yep.", + "Hello there.", + "What happened?", + "Try this now.", + "Please check again.", + "Start with the smallest change.", + "I'm listening.", + "I've got it.", + "Sounds good to me.", +]; + +fn main() -> Result<(), String> { + let mut args = std::env::args().skip(1); + let model_dir = PathBuf::from(args.next().ok_or("missing model directory argument")?); + let output = PathBuf::from(args.next().ok_or("missing output WAV argument")?); + let precision = match args.next().as_deref() { + None | Some("fp32") => PocketPrecision::Fp32, + Some("int8") => PocketPrecision::Int8, + Some(value) => return Err(format!("unsupported precision {value:?}; use fp32 or int8")), + }; + if args.next().is_some() { + return Err("unexpected extra argument".to_string()); + } + + let engine = load_text_to_speech_with_options( + model_dir + .to_str() + .ok_or_else(|| format!("model path is not UTF-8: {}", model_dir.display()))?, + PocketLoadOptions { + model: PocketModelSelection::English2026_04(precision), + num_threads: 1, + }, + )?; + eprintln!("precision: {precision:?}"); + let voice_path = model_dir.join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}")); + let voice = load_voice_style(&voice_path)?; + let boundary_silence = vec![0.0_f32; SAMPLE_RATE as usize / 4]; + let mut combined = Vec::new(); + + for (index, text) in PASSAGE.iter().enumerate() { + eprintln!("boundary {}: {text}", index + 1); + let samples = engine.synth_chunk(text, "en", &voice, 1)?; + eprintln!( + " raw samples: {} ({:.3} s)", + samples.len(), + samples.len() as f32 / SAMPLE_RATE as f32 + ); + combined.extend_from_slice(&samples); + if index + 1 != PASSAGE.len() { + combined.extend_from_slice(&boundary_silence); + } + } + + let output_str = output + .to_str() + .ok_or_else(|| format!("output path is not UTF-8: {}", output.display()))?; + if !sherpa_onnx::write(output_str, &combined, SAMPLE_RATE as i32) { + return Err(format!("could not write {}", output.display())); + } + eprintln!( + "wrote {} raw samples ({:.3} s) to {}", + combined.len(), + combined.len() as f32 / SAMPLE_RATE as f32, + output.display() + ); + Ok(()) +} diff --git a/crates/buzz-voice/src/lib.rs b/crates/buzz-voice/src/lib.rs new file mode 100644 index 0000000000..6d46e03de1 --- /dev/null +++ b/crates/buzz-voice/src/lib.rs @@ -0,0 +1,4 @@ +//! Reusable local voice primitives for Buzz. + +pub mod pocket; +pub mod preparation; diff --git a/crates/buzz-voice/src/pocket.rs b/crates/buzz-voice/src/pocket.rs new file mode 100644 index 0000000000..7d2da991bb --- /dev/null +++ b/crates/buzz-voice/src/pocket.rs @@ -0,0 +1,894 @@ +//! Pocket TTS engine for caller-provided January and April model bundles. +//! +//! The January layout runs through sherpa-onnx's `OfflineTts`. The April +//! `english_2026-04` bundle uses the same linked ONNX Runtime with its +//! SentencePiece tokenizer, learned BOS, and recurrent state manifests. +//! +//! ## Attribution +//! +//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. +//! arXiv:2509.06926. Original repository: . +//! Licensed CC-BY-4.0. +//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. +//! - **ONNX export**: KevinAHM — +//! . CC-BY-4.0. +//! - **sherpa-onnx repackage**: csukuangfj / k2-fsa — +//! . +//! Repackages KevinAHM's export with the file layout sherpa-onnx's +//! `OfflineTtsPocketModelConfig` expects. CC-BY-4.0. +//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary +//! (f, conversation)" preset from the Kyutai TTS demo +//! (), which maps to `vctk/p333_023_enhanced.wav` +//! in . CC-BY-4.0, base recording +//! from the VCTK corpus, enhanced by ai-coustics. +//! +//! Callers own acquisition, storage, and the required CC-BY-4.0 attribution. +//! +//! ## Engine-module contract (see `huddle::tts`) +//! +//! `pocket.rs` exposes the shared loading and synthesis surface, including +//! explicit runtime-only model options: +//! +//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. +//! - `DEFAULT_VOICE: &str` — default voice name (without extension). +//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. +//! - `load_text_to_speech(model_dir)` → `Result` +//! - `load_voice_style(path)` → `Result` +//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` +//! → `Result, String>` +//! +//! `lang` and `steps` are accepted for API compatibility with the previous +//! Kokoro engine but are unused — Pocket TTS does its own language ID from +//! the input text and is not a diffusion model (consistency LM, one step). +//! There is no speed knob: sherpa-onnx's `GenerationConfig.speed` is only +//! read by some model families (vits), never by the Pocket impl +//! (`offline-tts-pocket-impl.h` — zero references), and upstream pocket-tts +//! has no speed parameter either. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; + +#[path = "pocket_april.rs"] +mod pocket_april; +#[path = "pocket_models.rs"] +mod pocket_models; + +use pocket_april::{prepare_april_prompt, AprilPocketTts}; +pub use pocket_models::{ + april_model_info, PocketLoadOptions, PocketModelArtifact, PocketModelInfo, + PocketModelSelection, PocketPrecision, APRIL_BUNDLE_ID, APRIL_MAX_TOKEN_PER_CHUNK, + APRIL_MODEL_ID, APRIL_MODEL_REVISION, +}; + +// ── Engine-module contract: public consts ───────────────────────────────────── + +/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, +/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. +pub const SAMPLE_RATE: u32 = 24_000; + +/// Name (without extension) of the bundled reference voice. The model directory +/// is expected to contain `.` after install. +pub const DEFAULT_VOICE: &str = "reference_sample"; + +/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the +/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, +/// not a precomputed embedding, so the extension change is honest. +pub const VOICE_FILE_EXT: &str = "wav"; + +// ── Tuning ──────────────────────────────────────────────────────────────────── + +/// LRU cache size for cloned voice embeddings inside the sherpa-onnx engine. +/// We bind to one voice per pipeline today, but the upstream example uses 16 +/// and the cost is negligible — keep room for future multi-voice support. +const VOICE_EMBEDDING_CACHE_CAPACITY: i32 = 16; + +/// Pocket TTS is a consistency-based LM. Generation quality saturates at one +/// denoising step — the upstream `GenerationConfig` default of 5 multiplies +/// synthesis time by ~5× with no audible benefit on this model. +const SYNTH_NUM_STEPS: i32 = 1; + +/// Leave the generated audio's silences untouched (1.0 is the identity). +/// +/// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding +/// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and +/// multiplies its length by this factor. The reference Pocket TTS pipeline +/// preserves natural clause breaks, breaths, and punctuation pauses, so the +/// identity scale keeps those interior silences intact. +const SYNTH_SILENCE_SCALE: f32 = 1.0; + +/// sherpa-onnx upstream default for `max_frames` (LM steps), in +/// `offline-tts-pocket-impl.h:Generate`. 500 steps ≈ 40 s of audio at the +/// Mimi 12.5 Hz frame rate. Referenced only by the regression test below; +/// production code path never raises (or even reads) this value — we just +/// leave sherpa-onnx's own default in place by not setting the override. +#[cfg(test)] +const SHERPA_ONNX_MAX_FRAMES_DEFAULT: i32 = 500; + +/// Tight `max_frames` we ask for on short, padded prompts to bound the +/// original "monster breathing" runaway. 100 LM steps ≈ 8 s of audio — +/// roomy for any one-to-four-word utterance the user is likely to elicit +/// while still well short of the 40 s upstream default. Chosen with slack so +/// we never *truncate* a legitimate short reply. +const SHORT_PROMPT_MAX_FRAMES: i32 = 100; + +/// Word-count threshold (inclusive) below which we pad the prompt with +/// leading spaces and cap `max_frames` tighter than the upstream default. +/// Matches upstream `pocket_tts.models.tts_model.prepare_text_prompt`. Above +/// this threshold we leave sherpa-onnx's own defaults in place because +/// dropping `frames_after_eos` below the upstream default of 3 can clip the +/// leading audio of multi-clause sentences. +const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; + +/// Number of leading spaces prepended to short prompts. The upstream Python +/// uses exactly 8 — keep parity rather than tuning blindly. +/// +/// This is upstream's *only* mitigation for the FlowLM cold-start smear on +/// short utterances (kyutai-labs/pocket-tts #91, #70): the autoregressive +/// generation has a 2–3 step "settle" period where the first phoneme can be +/// smeared. The pad must remain whitespace-only: synthesized sacrificial text +/// requires amplitude-threshold trimming, which can eat soft word starts. +const SHORT_PROMPT_PAD_SPACES: usize = 8; + +/// sherpa-onnx's documented `frames_after_eos` default. We deliberately do +/// *not* override this knob because values below the upstream default of 3 can +/// clip the leading audio of multi-clause sentences. The constant exists only +/// for the invariant test below. Source: `offline-tts-pocket-impl.h:Generate`. +#[cfg(test)] +const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; + +// ── ONNX file names (five Pocket TTS sessions plus two JSON tables) ─────────── + +const FILE_LM_MAIN: &str = "lm_main.onnx"; +const FILE_LM_FLOW: &str = "lm_flow.onnx"; +const FILE_ENCODER: &str = "encoder.onnx"; +const FILE_DECODER: &str = "decoder.onnx"; +const FILE_TEXT_COND: &str = "text_conditioner.onnx"; +const FILE_VOCAB: &str = "vocab.json"; +const FILE_TOKEN_SCORES: &str = "token_scores.json"; + +// ── Voice style ─────────────────────────────────────────────────────────────── + +/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. +/// +/// Pocket TTS takes a reference waveform per generation call (not a +/// precomputed style embedding), so we keep the samples in memory and clone +/// the small `Vec` into each `GenerationConfig` rather than re-reading the +/// WAV from disk on every sentence. +#[derive(Debug, Clone)] +pub struct VoiceStyle { + samples: Vec, + sample_rate: i32, +} + +/// Load a reference voice WAV from disk. +/// +/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS +/// resamples internally using `reference_sample_rate`. The bundled +/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. +pub fn load_voice_style(path: &Path) -> Result { + let path_str = path + .to_str() + .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; + let wave = Wave::read(path_str) + .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; + let samples = wave.samples().to_vec(); + if samples.is_empty() { + return Err(format!("voice WAV is empty: {}", path.display())); + } + Ok(VoiceStyle { + samples, + sample_rate: wave.sample_rate(), + }) +} + +// ── Engine ──────────────────────────────────────────────────────────────────── + +/// Pocket TTS engine handle, normally kept resident by its caller. +pub struct PocketTts { + inner: PocketTtsInner, + model_info: Option, +} + +enum PocketTtsInner { + January(OfflineTts), + April(Box>), +} + +/// Build a Pocket TTS engine from a caller-provided model directory. +/// +/// An April `bundle.json` selects FP32 April automatically. Directories +/// without that manifest retain the source-compatible January loader. +pub fn load_text_to_speech(model_dir: &str) -> Result { + load_text_to_speech_with_options(model_dir, PocketLoadOptions::default()) +} + +/// Build a Pocket TTS engine with explicit runtime-only model options. +/// +/// Application download, cache, fallback, and UI policy intentionally remain +/// outside this crate. +pub fn load_text_to_speech_with_options( + model_dir: &str, + options: PocketLoadOptions, +) -> Result { + if options.num_threads == 0 { + return Err("Pocket TTS num_threads must be at least 1".to_string()); + } + let dir = PathBuf::from(model_dir); + let april_precision = match options.model { + PocketModelSelection::Auto if dir.join("bundle.json").is_file() => { + Some(PocketPrecision::Fp32) + } + PocketModelSelection::Auto => None, + PocketModelSelection::English2026_04(precision) => Some(precision), + }; + + if let Some(precision) = april_precision { + let info = april_model_info(precision); + for artifact in info.artifacts { + let path = dir.join(artifact.filename); + if !path.is_file() { + return Err(format!( + "incomplete Pocket TTS {} {:?} variant: missing {}", + info.bundle_id, + info.precision, + path.display() + )); + } + } + return Ok(PocketTts { + inner: PocketTtsInner::April(Box::new(Mutex::new(AprilPocketTts::load( + &dir, + precision, + options.num_threads, + )?))), + model_info: Some(info), + }); + } + + for name in [ + FILE_LM_MAIN, + FILE_LM_FLOW, + FILE_ENCODER, + FILE_DECODER, + FILE_TEXT_COND, + FILE_VOCAB, + FILE_TOKEN_SCORES, + ] { + let p = dir.join(name); + if !p.is_file() { + return Err(format!("missing Pocket TTS file: {}", p.display())); + } + } + + let to_str = |name: &str| -> String { dir.join(name).to_string_lossy().into_owned() }; + + // Build the config by mutating defaults — mirrors `stt.rs` and stays + // resilient if sherpa-onnx adds unrelated model-family fields. + let mut cfg = OfflineTtsConfig::default(); + cfg.model.pocket.lm_main = Some(to_str(FILE_LM_MAIN)); + cfg.model.pocket.lm_flow = Some(to_str(FILE_LM_FLOW)); + cfg.model.pocket.encoder = Some(to_str(FILE_ENCODER)); + cfg.model.pocket.decoder = Some(to_str(FILE_DECODER)); + cfg.model.pocket.text_conditioner = Some(to_str(FILE_TEXT_COND)); + cfg.model.pocket.vocab_json = Some(to_str(FILE_VOCAB)); + cfg.model.pocket.token_scores_json = Some(to_str(FILE_TOKEN_SCORES)); + cfg.model.pocket.voice_embedding_cache_capacity = VOICE_EMBEDDING_CACHE_CAPACITY; + cfg.model.num_threads = i32::try_from(options.num_threads).map_err(|_| { + format!( + "Pocket TTS num_threads is too large: {}", + options.num_threads + ) + })?; + // Explicit — defaults are not part of the API contract, and noisy debug + // logging in release builds would be expensive on every synthesized chunk. + cfg.model.debug = false; + + let inner = OfflineTts::create(&cfg) + .ok_or_else(|| "OfflineTts::create returned None for Pocket TTS".to_string())?; + Ok(PocketTts { + inner: PocketTtsInner::January(inner), + model_info: None, + }) +} + +// ── Prompt preparation ──────────────────────────────────────────────────────── + +/// Result of [`prepare_pocket_prompt`]: a synthesizer-ready prompt plus the +/// per-call generation overrides derived from the original text. +/// +/// `None` for either override means "leave sherpa-onnx's documented default +/// in place". The pipeline only sets `max_frames` (and only for short +/// padded inputs) so it can bound the original "monster breathing" runaway +/// without disturbing the rest of the LM sampling envelope. +#[derive(Debug, Clone, PartialEq)] +#[doc(hidden)] +pub struct PreparedPrompt { + /// Text to hand to `OfflineTts::generate_with_config`. Capitalized, + /// punctuation-terminated, and (for short inputs) left-padded with + /// spaces — upstream's mitigation for the FlowLM cold-start smear. + pub text: String, + /// Value to pass via `GenerationConfig.extra["max_frames"]`, or `None` to + /// keep the upstream default of 500 LM steps. We only override on short + /// padded prompts where we have a tight expectation on output length. + pub max_frames: Option, +} + +/// Mirror of the *text-preparation* half of upstream +/// `pocket_tts.models.tts_model.prepare_text_prompt`. Sherpa-onnx's C++ +/// Pocket TTS impl does not run these preparation steps, so short / +/// unpunctuated / lowercase inputs can trigger up to 40 s of runaway +/// generation when the EOS logit never crosses its threshold. We replicate +/// the upstream Python recipe here: +/// +/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but +/// cheap to re-check after sentence splitting). +/// 2. Capitalize the first letter. +/// 3. Append `.` if the text doesn't end in punctuation. +/// 4. If fewer than five words, prepend `SHORT_PROMPT_PAD_SPACES` spaces +/// (upstream's cold-start mitigation — see the constant's docstring) and +/// return a tight [`SHORT_PROMPT_MAX_FRAMES`] cap so the LM can't run +/// away if EOS still doesn't fire. +/// +/// We do **not** override `frames_after_eos` — sherpa-onnx's default of 3 +/// is what we want. An earlier version set it to 1 on long inputs, which +/// clipped the leading audio of multi-clause sentences ("first 'yep' is +/// just static" regression). Tests `prepare_prompt_never_lowers_frames_…` +/// lock this in. +/// +/// Returns `None` only if the input is empty after trimming — caller should +/// skip synthesis in that case. +#[doc(hidden)] +pub fn prepare_pocket_prompt(input: &str) -> Option { + let trimmed = input.trim(); + if trimmed.is_empty() { + return None; + } + + // Collapse stray double-spaces / embedded newlines that may slip past + // `preprocess_for_tts` when sentences are spliced back together. + let mut cleaned = String::with_capacity(trimmed.len()); + let mut last_was_space = false; + for ch in trimmed.chars() { + let is_ws = ch.is_whitespace(); + if is_ws { + if !last_was_space { + cleaned.push(' '); + } + last_was_space = true; + } else { + cleaned.push(ch); + last_was_space = false; + } + } + + // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). + let first = cleaned.chars().next().expect("cleaned non-empty above"); + if first.is_lowercase() { + let upper: String = first.to_uppercase().collect(); + let mut iter = cleaned.chars(); + iter.next(); + cleaned = upper + iter.as_str(); + } + + // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. + // The upstream Python only checks `isalnum` → period, but for our agent + // text we already may end in `!` `?` `.` etc. — treat any of those as OK. + let last = cleaned + .chars() + .next_back() + .expect("cleaned non-empty above"); + if !matches!(last, '.' | '!' | '?' | ';' | ':' | ',') { + cleaned.push('.'); + } + + // Word count of the *cleaned but not padded* text — padding is whitespace + // only and would just lie to the threshold check below. + let word_count = cleaned.split_whitespace().count(); + + let (final_text, max_frames) = if word_count <= SHORT_PROMPT_WORD_THRESHOLD { + let mut padded = String::with_capacity(cleaned.len() + SHORT_PROMPT_PAD_SPACES); + for _ in 0..SHORT_PROMPT_PAD_SPACES { + padded.push(' '); + } + padded.push_str(&cleaned); + (padded, Some(SHORT_PROMPT_MAX_FRAMES)) + } else { + // For everything ≥5 words, fall back to upstream defaults. Overriding + // these is what caused the "first 'yep' is static" regression — the + // upstream LM has been tuned for `frames_after_eos = 3` and + // `max_frames = 500`, and there's no clear win in second-guessing. + (cleaned, None) + }; + + Some(PreparedPrompt { + text: final_text, + max_frames, + }) +} + +/// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. +/// +/// Centralised so the invariant test below can assert that we **never** emit a +/// `frames_after_eos` override. Leaving the key unset preserves sherpa-onnx's +/// upstream default of 3. +fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { + prepared.max_frames.map(|mf| { + let mut h: HashMap = HashMap::with_capacity(1); + h.insert("max_frames".to_string(), serde_json::Value::from(mf)); + h + }) +} + +impl PocketTts { + /// Metadata for the loaded April model. + /// + /// January's legacy sherpa layout predates the immutable bundle contract, + /// so it returns `None`. + pub fn model_info(&self) -> Option { + self.model_info + } + + /// Prepare model-valid synthesis units for `text`. + /// + /// April splits with its bundle tokenizer and exact 50-token constraint. + /// The legacy January loader has no equivalent public tokenizer or token + /// limit, so it returns the prepared prompt as one synthesis unit. + pub fn split_text_into_chunks(&self, text: &str) -> Result, String> { + match &self.inner { + PocketTtsInner::January(_) => Ok(prepare_pocket_prompt(text) + .map(|prepared| vec![prepared.text]) + .unwrap_or_default()), + PocketTtsInner::April(engine) => { + let Some(prepared) = prepare_april_prompt(text) else { + return Ok(Vec::new()); + }; + engine + .lock() + .map_err(|_| "Pocket TTS engine lock poisoned".to_string())? + .split_prompt(&prepared) + } + } + } + + /// Synthesise `text` with the given reference voice. + /// + /// `_lang` and `_steps` are accepted for API compatibility with the + /// previous Kokoro engine. Pocket TTS infers language from the input text + /// directly and is a one-step consistency model. Returns an empty buffer + /// for whitespace-only input. + pub fn synth_chunk( + &self, + text: &str, + lang: &str, + style: &VoiceStyle, + steps: usize, + ) -> Result, String> { + self.synth_chunk_with_callback(text, lang, style, steps, None:: bool>) + } + + /// Synthesise `text`, allowing the caller to stop generation early. + /// + /// The callback receives the samples generated so far and a progress + /// value in `[0, 1]`. Return `true` to continue or `false` to cancel. + /// Clients that do not need cancellation should use [`Self::synth_chunk`]. + pub fn synth_chunk_with_callback( + &self, + text: &str, + _lang: &str, + style: &VoiceStyle, + _steps: usize, + mut callback: Option, + ) -> Result, String> + where + F: FnMut(&[f32], f32) -> bool + 'static, + { + match &self.inner { + PocketTtsInner::January(engine) => { + // Mirror January upstream prompt preparation and retain its + // short-prompt generation bounds. + let prepared = match prepare_pocket_prompt(text) { + Some(p) => p, + None => return Ok(Vec::new()), + }; + let cfg = GenerationConfig { + num_steps: SYNTH_NUM_STEPS, + silence_scale: SYNTH_SILENCE_SCALE, + reference_audio: Some(style.samples.clone()), + reference_sample_rate: style.sample_rate, + extra: build_generation_extra(&prepared), + ..Default::default() + }; + let audio = engine + .generate_with_config(&prepared.text, &cfg, callback) + .ok_or_else(|| { + format!( + "Pocket TTS synthesis failed for text ({} chars)", + prepared.text.len() + ) + })?; + let sample_rate = audio.sample_rate(); + if sample_rate != SAMPLE_RATE as i32 { + eprintln!( + "buzz-voice: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ + (expected {SAMPLE_RATE}Hz); playback speed may be wrong" + ); + } + Ok(audio.samples().to_vec()) + } + PocketTtsInner::April(engine) => { + let prepared = match prepare_april_prompt(text) { + Some(prepared) => prepared, + None => return Ok(Vec::new()), + }; + let mut engine = engine + .lock() + .map_err(|_| "Pocket TTS engine lock poisoned".to_string())?; + let chunks = engine.split_prompt(&prepared)?; + let chunk_count = chunks.len() as f32; + let mut samples = Vec::new(); + + for (chunk_index, chunk) in chunks.into_iter().enumerate() { + let prepared = prepare_april_prompt(&chunk) + .ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?; + let (chunk_samples, cancelled) = { + let mut cancelled = false; + let mut adapted_callback = + |current_chunk_samples: &[f32], chunk_progress: f32| { + let Some(callback) = callback.as_mut() else { + return true; + }; + let mut cumulative = + Vec::with_capacity(samples.len() + current_chunk_samples.len()); + cumulative.extend_from_slice(&samples); + cumulative.extend_from_slice(current_chunk_samples); + let progress = (chunk_index as f32 + chunk_progress) / chunk_count; + let should_continue = callback(&cumulative, progress); + cancelled = !should_continue; + should_continue + }; + let chunk_samples = + engine.synth_chunk(&prepared, style, Some(&mut adapted_callback))?; + (chunk_samples, cancelled) + }; + if cancelled { + return Ok(Vec::new()); + } + samples.extend(chunk_samples); + } + + Ok(samples) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── prepare_pocket_prompt ──────────────────────────────────────────────── + + #[test] + fn prepare_prompt_returns_none_for_empty_input() { + assert!(prepare_pocket_prompt("").is_none()); + assert!(prepare_pocket_prompt(" ").is_none()); + assert!(prepare_pocket_prompt("\n\t ").is_none()); + } + + /// Helper: the exact leading sequence prepended to every short prompt — + /// 8 spaces of padding (upstream's cold-start mitigation). + /// Centralising this keeps the assertions readable. + fn short_prefix() -> String { + " ".repeat(SHORT_PROMPT_PAD_SPACES) + } + + #[test] + fn prepare_prompt_pads_and_capitalizes_one_word() { + // The "yep" case Tyler hit in production — bare lowercase one-word + // utterance with no punctuation. Must be padded with the short-prompt + // space pad, capitalized, terminated, with a tight `max_frames` cap + // to bound runaway gen. + let out = prepare_pocket_prompt("yep").expect("non-empty"); + assert_eq!(out.text, format!("{}Yep.", short_prefix())); + assert_eq!(out.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); + const { + assert!( + SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT, + "short cap must be tighter than the upstream default" + ); + } + } + + #[test] + fn prepare_prompt_preserves_existing_punctuation() { + let out = prepare_pocket_prompt("yes!").expect("non-empty"); + assert_eq!(out.text, format!("{}Yes!", short_prefix())); // exclamation kept + let out = prepare_pocket_prompt("really?").expect("non-empty"); + assert_eq!(out.text, format!("{}Really?", short_prefix())); + } + + #[test] + fn prepare_prompt_threshold_is_inclusive_at_four_words() { + // 4 words = short (padded + tight max_frames); 5 words = long + // (no padding, no overrides — upstream defaults stand). + let four = prepare_pocket_prompt("one two three four").expect("non-empty"); + assert_eq!( + four.text, + format!("{}One two three four.", short_prefix()), + "four-word input should get exactly the space pad" + ); + assert_eq!(four.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); + + let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); + assert!( + !five.text.starts_with(' '), + "five-word input should NOT be padded" + ); + assert_eq!( + five.max_frames, None, + "long inputs must leave sherpa-onnx's max_frames default in place" + ); + } + + #[test] + fn prepare_prompt_does_not_pad_long_text() { + let long = "This is a longer sentence that the model should handle just fine."; + let out = prepare_pocket_prompt(long).expect("non-empty"); + assert!(!out.text.starts_with(' ')); + assert_eq!(out.max_frames, None); + assert!(out.text.ends_with('.')); + } + + #[test] + fn prepare_prompt_collapses_whitespace() { + let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); + // 3 words → short → padded. Interior whitespace collapsed. + assert_eq!(out.text, format!("{}Hello world friend.", short_prefix())); + } + + #[test] + fn prepare_prompt_does_not_double_capitalize_already_uppercase() { + let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); + assert_eq!(out.text, format!("{}HELLO there.", short_prefix())); + } + + #[test] + fn prepare_prompt_handles_non_ascii_first_letter() { + // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce + // mojibake. + let out = prepare_pocket_prompt("дa").expect("non-empty"); + assert!(out.text.contains("Дa.")); + } + + /// REGRESSION GUARD: short prompts must receive *only* whitespace + /// padding — no sacrificial text. A previous revision prepended a + /// `". . "` cold-start absorber and trimmed the rendered audio back out + /// with an amplitude threshold that could eat soft word onsets. If + /// non-whitespace ever reappears in the pad, the synth output will + /// contain audio for text the user never wrote. + #[test] + fn prepare_prompt_pad_is_whitespace_only() { + let out = prepare_pocket_prompt("I'm happy.").expect("non-empty"); + let pad_len = out.text.len() - "I'm happy.".len(); + assert!( + out.text[..pad_len].chars().all(|c| c == ' '), + "short-prompt pad must be spaces only, got {:?}", + &out.text[..pad_len] + ); + assert_eq!(out.text, format!("{}I'm happy.", short_prefix())); + } + + // ── build_generation_extra ─────────────────────────────────────────────── + // + // Short prompts override only `max_frames`; long prompts emit no extras. + // Every other knob remains at sherpa-onnx's documented default, notably + // `frames_after_eos = 3`. + + #[test] + fn build_extra_short_prompt_sets_only_max_frames() { + let prepared = prepare_pocket_prompt("yep").expect("non-empty"); + let extra = build_generation_extra(&prepared).expect("short prompts get extra"); + // Exactly one key — `max_frames` — and nothing else. + assert_eq!(extra.len(), 1, "extra has unexpected keys: {extra:?}"); + assert_eq!( + extra.get("max_frames"), + Some(&serde_json::Value::from(SHORT_PROMPT_MAX_FRAMES)) + ); + assert!( + !extra.contains_key("frames_after_eos"), + "frames_after_eos must never be set — upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT} is what we want" + ); + } + + #[test] + fn build_extra_long_prompt_is_none() { + // ≥5 words: no extras, so the upstream LM defaults remain authoritative. + let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); + assert_eq!( + build_generation_extra(&prepared), + None, + "long prompts must not override any LM knob" + ); + } + + #[test] + fn build_extra_never_lowers_frames_after_eos_for_any_word_count() { + // Sweep a range of prompt lengths and assert the `extra` map (when + // present) never carries a `frames_after_eos` override that's lower + // than the upstream sherpa-onnx default. Implemented as a structural + // check — we just never set the key — but worth a property test in + // case someone reintroduces the override in the future. + let prompts: &[&str] = &[ + "hi", + "hi there", + "yes please", + "one two three four", + "one two three four five", + "a slightly longer reply, hopefully fine", + "This is a multi-clause sentence. It has two parts.", + "really really really really really long prompt with lots of words just to be sure", + ]; + for &p in prompts { + let prepared = prepare_pocket_prompt(p).expect("non-empty"); + if let Some(extra) = build_generation_extra(&prepared) { + if let Some(v) = extra.get("frames_after_eos") { + let n = v.as_i64().expect("frames_after_eos should be int"); + assert!( + n >= SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT as i64, + "prompt {p:?} set frames_after_eos={n}, below upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT}" + ); + } + } + } + } + + #[test] + fn short_prompt_max_frames_is_below_upstream_default() { + // Sanity: the override only ever *lowers* the cap, never raises it. + const { + assert!(SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT); + } + // …and is still large enough for a one-to-four-word reply. At Mimi's + // 12.5 Hz frame rate, 100 frames = 8 s, which is roomy. + const { + assert!(SHORT_PROMPT_MAX_FRAMES >= 50, "would risk truncation"); + } + } + + #[test] + fn explicit_april_variant_reports_first_missing_artifact() { + let dir = + std::env::temp_dir().join(format!("buzz-pocket-incomplete-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("create incomplete model dir"); + + let error = load_text_to_speech_with_options( + dir.to_str().expect("UTF-8 temp path"), + PocketLoadOptions { + model: PocketModelSelection::English2026_04(PocketPrecision::Int8), + num_threads: 1, + }, + ) + .err() + .expect("incomplete variant should fail"); + + assert!(error.contains("incomplete Pocket TTS english_2026-04 Int8 variant")); + assert!(error.contains("bundle.json")); + std::fs::remove_dir_all(dir).expect("remove incomplete model dir"); + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"] + fn april_callback_can_cancel_before_model_inference() { + let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle"); + let engine = load_text_to_speech_with_options( + &dir, + PocketLoadOptions { + model: PocketModelSelection::English2026_04(PocketPrecision::Fp32), + num_threads: 1, + }, + ) + .expect("load April FP32 bundle"); + let voice = + load_voice_style(&Path::new(&dir).join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}"))) + .expect("load reference voice"); + + let samples = engine + .synth_chunk_with_callback( + "Cancel before inference.", + "en", + &voice, + 1, + Some(|_samples: &[f32], _progress: f32| false), + ) + .expect("cancel synthesis"); + assert!(samples.is_empty()); + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR with FP32 and INT8 artifacts"] + fn april_precisions_synthesize_non_silent_audio() { + let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle"); + let voice = + load_voice_style(&Path::new(&dir).join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}"))) + .expect("load reference voice"); + + for precision in [PocketPrecision::Fp32, PocketPrecision::Int8] { + let engine = load_text_to_speech_with_options( + &dir, + PocketLoadOptions { + model: PocketModelSelection::English2026_04(precision), + num_threads: 1, + }, + ) + .expect("load April precision"); + assert_eq!( + engine.model_info().expect("April model info").precision, + precision + ); + let samples = engine + .synth_chunk("Hello there.", "en", &voice, 1) + .expect("synthesize April precision"); + assert!(!samples.is_empty(), "{precision:?} returned no samples"); + assert!( + samples.iter().any(|sample| sample.abs() > 1e-5), + "{precision:?} returned digital silence" + ); + } + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"] + fn april_public_synthesis_handles_text_over_token_limit() { + let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle"); + let engine = load_text_to_speech_with_options( + &dir, + PocketLoadOptions { + model: PocketModelSelection::English2026_04(PocketPrecision::Fp32), + num_threads: 1, + }, + ) + .expect("load April FP32 bundle"); + let voice = + load_voice_style(&Path::new(&dir).join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}"))) + .expect("load reference voice"); + let text = "This deliberately long passage crosses the model token boundary while using \ + the public synthesis method directly. It verifies that callers receive one \ + continuous sample buffer without needing to duplicate the model tokenizer or \ + understand its exact token budget. The runtime must split the prompt safely, \ + synthesize every piece, and preserve compatibility with existing callers."; + + let chunks = engine + .split_text_into_chunks(text) + .expect("split long April prompt"); + assert!(chunks.len() > 1, "test text must exceed one model chunk"); + + let samples = engine + .synth_chunk(text, "en", &voice, 1) + .expect("synthesize long April prompt"); + assert!(!samples.is_empty()); + assert!(samples.iter().any(|sample| sample.abs() > 1e-5)); + } + + #[test] + #[ignore = "requires BUZZ_POCKET_JANUARY_TEST_MODEL_DIR"] + fn january_auto_detection_remains_source_compatible() { + let dir = std::env::var("BUZZ_POCKET_JANUARY_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_JANUARY_TEST_MODEL_DIR to a January bundle"); + let engine = load_text_to_speech(&dir).expect("auto-load January bundle"); + assert!(engine.model_info().is_none()); + let voice = + load_voice_style(&Path::new(&dir).join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}"))) + .expect("load January reference voice"); + let samples = engine + .synth_chunk("Hello there.", "en", &voice, 1) + .expect("synthesize January bundle"); + assert!(!samples.is_empty()); + assert!(samples.iter().any(|sample| sample.abs() > 1e-5)); + } +} diff --git a/crates/buzz-voice/src/pocket_april.rs b/crates/buzz-voice/src/pocket_april.rs new file mode 100644 index 0000000000..a2ce46d934 --- /dev/null +++ b/crates/buzz-voice/src/pocket_april.rs @@ -0,0 +1,966 @@ +//! Native ONNX loader for Pocket TTS `english_2026-04`. +//! +//! sherpa-onnx 1.13.4 remains Buzz's shared speech runtime, but its Pocket +//! frontend only understands the January vocab/token-score bundle. The April +//! export uses SentencePiece, prepends a learned BOS voice embedding, and +//! describes recurrent state tensors in `bundle.json`. This module implements +//! only that missing frontend/state loop and reuses the ONNX Runtime already +//! linked by sherpa-onnx. + +use std::borrow::Cow; +use std::f32::consts::TAU; +use std::fs; +use std::path::{Path, PathBuf}; + +use ort::session::{Session, SessionInputValue}; +use ort::value::{DynValue, Tensor}; +use rand::{Rng, RngExt}; +use sentencepiece_model::SentencePieceModel; +use serde::Deserialize; +use sherpa_onnx::LinearResampler; +use tokenizers::models::unigram::Unigram; +use tokenizers::pre_tokenizers::metaspace::{Metaspace, PrependScheme}; +use tokenizers::Tokenizer; + +use super::{PocketPrecision, VoiceStyle}; + +const FILE_BUNDLE: &str = "bundle.json"; +const FILE_MIMI_ENCODER: &str = "mimi_encoder.onnx"; +const FILE_TEXT_CONDITIONER: &str = "text_conditioner.onnx"; +const FILE_FLOW_MAIN_FP32: &str = "flow_lm_main.onnx"; +const FILE_FLOW_FP32: &str = "flow_lm_flow.onnx"; +const FILE_MIMI_DECODER_FP32: &str = "mimi_decoder.onnx"; +const FILE_FLOW_MAIN_INT8: &str = "flow_lm_main_int8.onnx"; +const FILE_FLOW_INT8: &str = "flow_lm_flow_int8.onnx"; +const FILE_MIMI_DECODER_INT8: &str = "mimi_decoder_int8.onnx"; + +const MODEL_LANGUAGE: &str = "english_2026-04"; +const DEFAULT_TEMPERATURE: f32 = 0.7; +const EOS_LOGIT_THRESHOLD: f32 = -4.0; +const DECODER_CHUNK_FRAMES: usize = 12; +const GENERATION_PROGRESS_SHARE: f32 = 0.9; +const TOKENS_PER_SECOND_ESTIMATE: f32 = 3.0; +const GENERATION_SECONDS_PADDING: f32 = 2.0; + +type ProgressCallback<'a> = Option<&'a mut dyn FnMut(&[f32], f32) -> bool>; + +#[derive(Debug, Deserialize)] +struct Bundle { + schema_version: u32, + language: String, + sample_rate: usize, + frame_rate: f32, + samples_per_frame: usize, + latent_dim: usize, + conditioning_dim: usize, + insert_bos_before_voice: bool, + pad_with_spaces_for_short_inputs: bool, + remove_semicolons: bool, + model_recommended_frames_after_eos: Option, + max_token_per_chunk: usize, + tokenizer_file: String, + bos_before_voice_file: String, + flow_lm_state_manifest: Vec, + mimi_state_manifest: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +struct StateSpec { + input_name: String, + output_name: String, + dtype: StateDtype, + shape: Vec, + fill: StateFill, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum StateDtype { + #[serde(rename = "float32")] + Float32, + #[serde(rename = "int64")] + Int64, + Bool, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum StateFill { + Empty, + Nan, + Ones, + Zeros, +} + +struct StateValue { + spec: StateSpec, + value: DynValue, +} + +struct CachedVoice { + samples_ptr: usize, + samples_len: usize, + sample_rate: i32, + embeddings: Vec, +} + +pub(crate) struct AprilPocketTts { + bundle: Bundle, + tokenizer: Tokenizer, + bos_embedding: Vec, + mimi_encoder: Session, + text_conditioner: Session, + flow_main: Session, + flow: Session, + mimi_decoder: Session, + cached_voice: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct AprilPreparedPrompt { + pub(crate) text: String, + pub(crate) frames_after_eos: usize, +} + +pub(crate) fn prepare_april_prompt(input: &str) -> Option { + let trimmed = input.trim(); + if trimmed.is_empty() { + return None; + } + + let mut cleaned = String::with_capacity(trimmed.len()); + let mut last_was_space = false; + for ch in trimmed.chars() { + if ch.is_whitespace() { + if !last_was_space { + cleaned.push(' '); + } + last_was_space = true; + } else { + cleaned.push(ch); + last_was_space = false; + } + } + + let first = cleaned.chars().next().expect("cleaned non-empty above"); + if first.is_lowercase() { + let upper: String = first.to_uppercase().collect(); + let mut iter = cleaned.chars(); + iter.next(); + cleaned = upper + iter.as_str(); + } + + let last = cleaned + .chars() + .next_back() + .expect("cleaned non-empty above"); + if last.is_alphanumeric() { + cleaned.push('.'); + } + + let word_count = cleaned.split_whitespace().count(); + Some(AprilPreparedPrompt { + text: cleaned, + // Mirror the bundle's upstream heuristic: three generated frames plus + // two trailing frames for short prompts, one plus two otherwise. + frames_after_eos: if word_count <= 4 { 5 } else { 3 }, + }) +} + +impl AprilPocketTts { + pub(crate) fn load( + dir: &Path, + precision: PocketPrecision, + num_threads: usize, + ) -> Result { + if num_threads == 0 { + return Err("Pocket TTS num_threads must be at least 1".to_string()); + } + let bundle_path = dir.join(FILE_BUNDLE); + let bundle: Bundle = serde_json::from_slice( + &fs::read(&bundle_path) + .map_err(|err| format!("read {}: {err}", bundle_path.display()))?, + ) + .map_err(|err| format!("parse {}: {err}", bundle_path.display()))?; + + if bundle.schema_version != 2 { + return Err(format!( + "unsupported Pocket TTS bundle schema {} in {}", + bundle.schema_version, + bundle_path.display() + )); + } + if bundle.language != MODEL_LANGUAGE { + return Err(format!( + "expected Pocket TTS language {MODEL_LANGUAGE}, got {}", + bundle.language + )); + } + if bundle.sample_rate != 24_000 + || bundle.frame_rate != 12.5 + || bundle.samples_per_frame != 1_920 + || bundle.latent_dim != 32 + || bundle.conditioning_dim != 1024 + { + return Err(format!( + "unexpected Pocket TTS dimensions: sample_rate={}, frame_rate={}, samples_per_frame={}, latent_dim={}, conditioning_dim={}", + bundle.sample_rate, + bundle.frame_rate, + bundle.samples_per_frame, + bundle.latent_dim, + bundle.conditioning_dim + )); + } + if !bundle.insert_bos_before_voice { + return Err("April Pocket TTS bundle must insert BOS before voice".to_string()); + } + if bundle.pad_with_spaces_for_short_inputs + || bundle.remove_semicolons + || bundle.model_recommended_frames_after_eos.is_some() + || bundle.max_token_per_chunk != 50 + { + return Err("unsupported April Pocket TTS prompt-policy metadata".to_string()); + } + + let tokenizer_path = dir.join(&bundle.tokenizer_file); + let tokenizer = load_tokenizer(&tokenizer_path)?; + let bos_path = dir.join(&bundle.bos_before_voice_file); + let bos_embedding = read_npy_f32(&bos_path)?; + if bos_embedding.len() != bundle.conditioning_dim { + return Err(format!( + "{} has {} values; expected {}", + bos_path.display(), + bos_embedding.len(), + bundle.conditioning_dim + )); + } + + let (flow_main, flow, mimi_decoder) = match precision { + PocketPrecision::Fp32 => (FILE_FLOW_MAIN_FP32, FILE_FLOW_FP32, FILE_MIMI_DECODER_FP32), + PocketPrecision::Int8 => (FILE_FLOW_MAIN_INT8, FILE_FLOW_INT8, FILE_MIMI_DECODER_INT8), + }; + + Ok(Self { + // Match the pinned upstream precision selector: voice encoding and + // text conditioning remain full precision in both modes. + mimi_encoder: load_session(dir.join(FILE_MIMI_ENCODER), num_threads)?, + text_conditioner: load_session(dir.join(FILE_TEXT_CONDITIONER), num_threads)?, + flow_main: load_session(dir.join(flow_main), num_threads)?, + flow: load_session(dir.join(flow), num_threads)?, + mimi_decoder: load_session(dir.join(mimi_decoder), num_threads)?, + bundle, + tokenizer, + bos_embedding, + cached_voice: None, + }) + } + + pub(crate) fn split_prompt( + &self, + prepared: &AprilPreparedPrompt, + ) -> Result, String> { + if self.token_count(&prepared.text)? <= self.bundle.max_token_per_chunk { + return Ok(vec![prepared.text.clone()]); + } + + let mut chunks = Vec::new(); + let mut current = String::new(); + for word in prepared.text.split_whitespace() { + let candidate = if current.is_empty() { + word.to_string() + } else { + format!("{current} {word}") + }; + if self.prepared_token_count(&candidate)? <= self.bundle.max_token_per_chunk { + current = candidate; + continue; + } + if !current.is_empty() { + chunks.push(std::mem::take(&mut current)); + } + + if self.prepared_token_count(word)? <= self.bundle.max_token_per_chunk { + current = word.to_string(); + continue; + } + + let mut fragment = String::new(); + for ch in word.chars() { + let candidate = format!("{fragment}{ch}"); + if !fragment.is_empty() + && self.prepared_token_count(&candidate)? > self.bundle.max_token_per_chunk + { + chunks.push(std::mem::take(&mut fragment)); + } + fragment.push(ch); + } + current = fragment; + } + if !current.is_empty() { + chunks.push(current); + } + + chunks + .into_iter() + .map(|text| { + let chunk = prepare_april_prompt(&text) + .ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?; + let token_count = self.token_count(&chunk.text)?; + if token_count > self.bundle.max_token_per_chunk { + return Err(format!( + "Pocket TTS prompt chunk has {token_count} tokens; maximum is {}", + self.bundle.max_token_per_chunk + )); + } + Ok(chunk.text) + }) + .collect() + } + + pub(crate) fn synth_chunk( + &mut self, + prepared: &AprilPreparedPrompt, + style: &VoiceStyle, + mut callback: ProgressCallback<'_>, + ) -> Result, String> { + if !should_continue(&mut callback, &[], 0.0) { + return Ok(Vec::new()); + } + + let voice_embeddings = self.voice_embeddings(style)?; + let mut flow_state = self.condition_voice(&voice_embeddings)?; + let token_ids = self + .tokenizer + .encode(prepared.text.as_str(), false) + .map_err(|err| format!("tokenize Pocket TTS prompt: {err}"))? + .get_ids() + .iter() + .copied() + .map(i64::from) + .collect::>(); + if token_ids.is_empty() { + return Ok(Vec::new()); + } + if token_ids.len() > self.bundle.max_token_per_chunk { + return Err(format!( + "Pocket TTS prompt has {} tokens; split_text_into_chunks maximum is {}", + token_ids.len(), + self.bundle.max_token_per_chunk + )); + } + + let token_count = token_ids.len(); + let text_embeddings = self.text_embeddings(token_ids)?; + self.run_flow_main_prefix(&text_embeddings, &mut flow_state)?; + let max_frames = estimate_max_frames(token_count, self.bundle.frame_rate); + let Some(latents) = self.generate_latents( + max_frames, + prepared.frames_after_eos, + &mut flow_state, + &mut callback, + )? + else { + return Ok(Vec::new()); + }; + Ok(self + .decode_latents(&latents, &mut callback)? + .unwrap_or_default()) + } + + fn prepared_token_count(&self, text: &str) -> Result { + let prepared = prepare_april_prompt(text) + .ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?; + self.token_count(&prepared.text) + } + + fn token_count(&self, text: &str) -> Result { + Ok(self + .tokenizer + .encode(text, false) + .map_err(|err| format!("tokenize Pocket TTS prompt: {err}"))? + .get_ids() + .len()) + } + + fn voice_embeddings(&mut self, style: &VoiceStyle) -> Result, String> { + let key = ( + style.samples.as_ptr() as usize, + style.samples.len(), + style.sample_rate, + ); + if let Some(cached) = &self.cached_voice { + if (cached.samples_ptr, cached.samples_len, cached.sample_rate) == key { + return Ok(cached.embeddings.clone()); + } + } + + let samples = if style.sample_rate == self.bundle.sample_rate as i32 { + style.samples.clone() + } else { + LinearResampler::create(style.sample_rate, self.bundle.sample_rate as i32) + .ok_or_else(|| { + format!( + "create Pocket TTS resampler {}Hz -> {}Hz", + style.sample_rate, self.bundle.sample_rate + ) + })? + .resample(&style.samples, true) + }; + let audio = Tensor::from_array(( + vec![1_i64, 1, samples.len() as i64], + samples.into_boxed_slice(), + )) + .map_err(ort_error("create voice audio tensor"))?; + let outputs = self + .mimi_encoder + .run(ort::inputs!["audio" => audio]) + .map_err(ort_error("run Mimi encoder"))?; + let (_, encoded) = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract Mimi encoder output"))?; + if !encoded.len().is_multiple_of(self.bundle.conditioning_dim) { + return Err(format!( + "Mimi encoder returned {} values, not divisible by {}", + encoded.len(), + self.bundle.conditioning_dim + )); + } + let mut embeddings = + Vec::with_capacity(self.bos_embedding.len().saturating_add(encoded.len())); + embeddings.extend_from_slice(&self.bos_embedding); + embeddings.extend_from_slice(encoded); + self.cached_voice = Some(CachedVoice { + samples_ptr: key.0, + samples_len: key.1, + sample_rate: key.2, + embeddings: embeddings.clone(), + }); + Ok(embeddings) + } + + fn condition_voice(&mut self, embeddings: &[f32]) -> Result, String> { + let frames = embeddings.len() / self.bundle.conditioning_dim; + let sequence = Tensor::::new( + &ort::memory::Allocator::default(), + [1_i64, 0, self.bundle.latent_dim as i64], + ) + .map_err(ort_error("create empty voice sequence"))?; + let text_embeddings = Tensor::from_array(( + vec![1_i64, frames as i64, self.bundle.conditioning_dim as i64], + embeddings.to_vec().into_boxed_slice(), + )) + .map_err(ort_error("create voice embedding tensor"))?; + let mut state = initialize_state(&self.bundle.flow_lm_state_manifest)?; + let mut inputs = vec![ + (Cow::Borrowed("sequence"), SessionInputValue::from(sequence)), + ( + Cow::Borrowed("text_embeddings"), + SessionInputValue::from(text_embeddings), + ), + ]; + append_state_inputs(&mut inputs, &state); + let mut outputs = self + .flow_main + .run(inputs) + .map_err(ort_error("condition Pocket TTS voice"))?; + replace_state_from_outputs(&mut state, &mut outputs)?; + Ok(state) + } + + fn text_embeddings(&mut self, token_ids: Vec) -> Result, String> { + let tokens = Tensor::from_array(( + vec![1_i64, token_ids.len() as i64], + token_ids.into_boxed_slice(), + )) + .map_err(ort_error("create token tensor"))?; + let outputs = self + .text_conditioner + .run(ort::inputs!["token_ids" => tokens]) + .map_err(ort_error("run text conditioner"))?; + let (_, embeddings) = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract text embeddings"))?; + Ok(embeddings.to_vec()) + } + + fn run_flow_main_prefix( + &mut self, + text_embeddings: &[f32], + state: &mut [StateValue], + ) -> Result<(), String> { + if !text_embeddings + .len() + .is_multiple_of(self.bundle.conditioning_dim) + { + return Err(format!( + "text conditioner returned {} values, not divisible by {}", + text_embeddings.len(), + self.bundle.conditioning_dim + )); + } + let frames = text_embeddings.len() / self.bundle.conditioning_dim; + let sequence = Tensor::::new( + &ort::memory::Allocator::default(), + [1_i64, 0, self.bundle.latent_dim as i64], + ) + .map_err(ort_error("create empty text sequence"))?; + let text_embeddings = Tensor::from_array(( + vec![1_i64, frames as i64, self.bundle.conditioning_dim as i64], + text_embeddings.to_vec().into_boxed_slice(), + )) + .map_err(ort_error("create text embedding tensor"))?; + let mut inputs = vec![ + (Cow::Borrowed("sequence"), SessionInputValue::from(sequence)), + ( + Cow::Borrowed("text_embeddings"), + SessionInputValue::from(text_embeddings), + ), + ]; + append_state_inputs(&mut inputs, state); + let mut outputs = self + .flow_main + .run(inputs) + .map_err(ort_error("prime Pocket TTS text state"))?; + replace_state_from_outputs(state, &mut outputs) + } + + fn generate_latents( + &mut self, + max_frames: usize, + frames_after_eos: usize, + state: &mut [StateValue], + callback: &mut ProgressCallback<'_>, + ) -> Result>, String> { + let mut current = vec![f32::NAN; self.bundle.latent_dim]; + let mut latents = Vec::with_capacity(max_frames * self.bundle.latent_dim); + let mut eos_step = None; + let mut rng = rand::rng(); + + for step in 0..max_frames { + let progress = GENERATION_PROGRESS_SHARE * (step as f32 / max_frames.max(1) as f32); + if !should_continue(callback, &[], progress) { + return Ok(None); + } + + let sequence = Tensor::from_array(( + vec![1_i64, 1, self.bundle.latent_dim as i64], + current.clone().into_boxed_slice(), + )) + .map_err(ort_error("create latent input"))?; + let text_embeddings = Tensor::::new( + &ort::memory::Allocator::default(), + [1_i64, 0, self.bundle.conditioning_dim as i64], + ) + .map_err(ort_error("create empty text input"))?; + let mut inputs = vec![ + (Cow::Borrowed("sequence"), SessionInputValue::from(sequence)), + ( + Cow::Borrowed("text_embeddings"), + SessionInputValue::from(text_embeddings), + ), + ]; + append_state_inputs(&mut inputs, state); + let mut outputs = self + .flow_main + .run(inputs) + .map_err(ort_error("run Pocket TTS Flow LM"))?; + let conditioning = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract Flow LM conditioning"))? + .1 + .to_vec(); + let eos_logit = outputs[1] + .try_extract_tensor::() + .map_err(ort_error("extract Flow LM EOS logit"))? + .1 + .first() + .copied() + .ok_or_else(|| "Flow LM returned empty EOS logit".to_string())?; + replace_state_from_outputs(state, &mut outputs)?; + + if eos_logit > EOS_LOGIT_THRESHOLD && eos_step.is_none() { + eos_step = Some(step); + } + if eos_step.is_some_and(|eos| step >= eos + frames_after_eos) { + break; + } + + let mut noise = + normal_noise(&mut rng, self.bundle.latent_dim, DEFAULT_TEMPERATURE.sqrt()); + let conditioning = Tensor::from_array(( + vec![1_i64, self.bundle.conditioning_dim as i64], + conditioning.into_boxed_slice(), + )) + .map_err(ort_error("create flow conditioning"))?; + let s = Tensor::from_array((vec![1_i64, 1], vec![0.0_f32].into_boxed_slice())) + .map_err(ort_error("create flow start tensor"))?; + let t = Tensor::from_array((vec![1_i64, 1], vec![1.0_f32].into_boxed_slice())) + .map_err(ort_error("create flow end tensor"))?; + let x = Tensor::from_array(( + vec![1_i64, self.bundle.latent_dim as i64], + noise.clone().into_boxed_slice(), + )) + .map_err(ort_error("create flow noise tensor"))?; + let outputs = self + .flow + .run(ort::inputs![ + "c" => conditioning, + "s" => s, + "t" => t, + "x" => x, + ]) + .map_err(ort_error("run Pocket TTS flow"))?; + let flow = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract Pocket TTS flow"))? + .1; + if flow.len() != noise.len() { + return Err(format!( + "flow returned {} values; expected {}", + flow.len(), + noise.len() + )); + } + for (sample, delta) in noise.iter_mut().zip(flow) { + *sample += *delta; + } + current.clone_from(&noise); + latents.extend_from_slice(&noise); + } + Ok(Some(latents)) + } + + fn decode_latents( + &mut self, + latents: &[f32], + callback: &mut ProgressCallback<'_>, + ) -> Result>, String> { + if latents.is_empty() { + return Ok(Some(Vec::new())); + } + if !latents.len().is_multiple_of(self.bundle.latent_dim) { + return Err(format!( + "latent buffer has {} values, not divisible by {}", + latents.len(), + self.bundle.latent_dim + )); + } + let frame_count = latents.len() / self.bundle.latent_dim; + let mut state = initialize_state(&self.bundle.mimi_state_manifest)?; + let mut audio = Vec::new(); + + for start in (0..frame_count).step_by(DECODER_CHUNK_FRAMES) { + let end = (start + DECODER_CHUNK_FRAMES).min(frame_count); + let values = + latents[start * self.bundle.latent_dim..end * self.bundle.latent_dim].to_vec(); + let latent = Tensor::from_array(( + vec![1_i64, (end - start) as i64, self.bundle.latent_dim as i64], + values.into_boxed_slice(), + )) + .map_err(ort_error("create Mimi latent tensor"))?; + let mut inputs = vec![(Cow::Borrowed("latent"), SessionInputValue::from(latent))]; + append_state_inputs(&mut inputs, &state); + let mut outputs = self + .mimi_decoder + .run(inputs) + .map_err(ort_error("run Mimi decoder"))?; + let samples = outputs[0] + .try_extract_tensor::() + .map_err(ort_error("extract Mimi audio"))? + .1; + audio.extend_from_slice(samples); + replace_state_from_outputs(&mut state, &mut outputs)?; + + let progress = GENERATION_PROGRESS_SHARE + + (1.0 - GENERATION_PROGRESS_SHARE) * (end as f32 / frame_count as f32); + if !should_continue(callback, &audio, progress) { + return Ok(None); + } + } + Ok(Some(audio)) + } +} + +fn should_continue(callback: &mut ProgressCallback<'_>, samples: &[f32], progress: f32) -> bool { + callback + .as_mut() + .is_none_or(|callback| callback(samples, progress.clamp(0.0, 1.0))) +} + +fn load_session(path: PathBuf, num_threads: usize) -> Result { + if !path.is_file() { + return Err(format!("missing Pocket TTS file: {}", path.display())); + } + Session::builder() + .map_err(ort_error("create ONNX session builder"))? + .with_intra_threads(num_threads) + .map_err(|err| format!("configure ONNX intra-op threads: {err}"))? + .with_inter_threads(1) + .map_err(|err| format!("configure ONNX inter-op threads: {err}"))? + .commit_from_file(&path) + .map_err(|err| format!("load {}: {err}", path.display())) +} + +fn load_tokenizer(path: &Path) -> Result { + let sentencepiece = SentencePieceModel::from_file(path) + .map_err(|err| format!("load {}: {err}", path.display()))?; + let trainer = sentencepiece + .trainer() + .ok_or_else(|| format!("{} has no SentencePiece trainer metadata", path.display()))?; + let normalizer = sentencepiece.normalizer().ok_or_else(|| { + format!( + "{} has no SentencePiece normalizer metadata", + path.display() + ) + })?; + if normalizer.name() != "identity" { + return Err(format!( + "{} uses unsupported SentencePiece normalizer {:?}", + path.display(), + normalizer.name() + )); + } + + let vocab = sentencepiece + .pieces() + .iter() + .map(|piece| (piece.piece().to_owned(), f64::from(piece.score()))) + .collect(); + let mut tokenizer = Tokenizer::new( + Unigram::from( + vocab, + Some(trainer.unk_id() as usize), + trainer.byte_fallback(), + ) + .map_err(|err| format!("construct tokenizer from {}: {err}", path.display()))?, + ); + // SentencePiece's identity normalizer still escapes spaces as U+2581 and + // prepends one marker to the input before unigram segmentation. + tokenizer.with_pre_tokenizer(Some(Metaspace::new('▁', PrependScheme::Always, false))); + Ok(tokenizer) +} + +fn initialize_state(specs: &[StateSpec]) -> Result, String> { + specs + .iter() + .cloned() + .map(|spec| { + let len = shape_len(&spec.shape)?; + let value = match spec.dtype { + StateDtype::Float32 => { + let fill = match spec.fill { + StateFill::Nan => f32::NAN, + StateFill::Empty | StateFill::Zeros => 0.0, + StateFill::Ones => 1.0, + }; + if len == 0 { + Tensor::::new(&ort::memory::Allocator::default(), spec.shape.clone()) + .map_err(ort_error("create empty float state tensor"))? + .into_dyn() + } else { + Tensor::from_array((spec.shape.clone(), vec![fill; len].into_boxed_slice())) + .map_err(ort_error("create float state tensor"))? + .into_dyn() + } + } + StateDtype::Int64 => { + let fill = i64::from(matches!(spec.fill, StateFill::Ones)); + if len == 0 { + Tensor::::new(&ort::memory::Allocator::default(), spec.shape.clone()) + .map_err(ort_error("create empty integer state tensor"))? + .into_dyn() + } else { + Tensor::from_array((spec.shape.clone(), vec![fill; len].into_boxed_slice())) + .map_err(ort_error("create integer state tensor"))? + .into_dyn() + } + } + StateDtype::Bool => { + let fill = matches!(spec.fill, StateFill::Ones); + if len == 0 { + Tensor::::new(&ort::memory::Allocator::default(), spec.shape.clone()) + .map_err(ort_error("create empty bool state tensor"))? + .into_dyn() + } else { + Tensor::from_array((spec.shape.clone(), vec![fill; len].into_boxed_slice())) + .map_err(ort_error("create bool state tensor"))? + .into_dyn() + } + } + }; + Ok(StateValue { spec, value }) + }) + .collect() +} + +fn append_state_inputs<'a>( + inputs: &mut Vec<(Cow<'a, str>, SessionInputValue<'a>)>, + state: &'a [StateValue], +) { + for value in state { + inputs.push(( + Cow::Borrowed(value.spec.input_name.as_str()), + SessionInputValue::from(&value.value), + )); + } +} + +fn replace_state_from_outputs( + state: &mut [StateValue], + outputs: &mut ort::session::SessionOutputs<'_>, +) -> Result<(), String> { + for value in state { + value.value = outputs + .remove(&value.spec.output_name) + .ok_or_else(|| format!("missing state output {}", value.spec.output_name))?; + } + Ok(()) +} + +fn shape_len(shape: &[i64]) -> Result { + shape.iter().try_fold(1_usize, |len, &dim| { + let dim = usize::try_from(dim).map_err(|_| format!("negative state dimension {dim}"))?; + len.checked_mul(dim) + .ok_or_else(|| format!("state shape overflows usize: {shape:?}")) + }) +} + +fn estimate_max_frames(token_count: usize, frame_rate: f32) -> usize { + ((token_count as f32 / TOKENS_PER_SECOND_ESTIMATE + GENERATION_SECONDS_PADDING) * frame_rate) + .ceil() as usize +} + +fn normal_noise(rng: &mut impl Rng, len: usize, std_dev: f32) -> Vec { + let mut out = Vec::with_capacity(len); + while out.len() < len { + let u1 = rng.random::().max(f32::MIN_POSITIVE); + let u2 = rng.random::(); + let radius = (-2.0_f32 * u1.ln()).sqrt() * std_dev; + out.push(radius * (TAU * u2).cos()); + if out.len() < len { + out.push(radius * (TAU * u2).sin()); + } + } + out +} + +fn read_npy_f32(path: &Path) -> Result, String> { + let bytes = fs::read(path).map_err(|err| format!("read {}: {err}", path.display()))?; + if bytes.len() < 10 || &bytes[..6] != b"\x93NUMPY" { + return Err(format!("{} is not a NumPy array", path.display())); + } + let major = bytes[6]; + let header_len_bytes = match major { + 1 => 2, + 2 | 3 => 4, + _ => { + return Err(format!( + "unsupported NumPy version {major} in {}", + path.display() + )) + } + }; + let header_start = 8 + header_len_bytes; + if bytes.len() < header_start { + return Err(format!("truncated NumPy header in {}", path.display())); + } + let header_len = if header_len_bytes == 2 { + u16::from_le_bytes([bytes[8], bytes[9]]) as usize + } else { + u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize + }; + let data_start = header_start + .checked_add(header_len) + .ok_or_else(|| format!("NumPy header overflow in {}", path.display()))?; + if data_start > bytes.len() { + return Err(format!("truncated NumPy data in {}", path.display())); + } + let header = std::str::from_utf8(&bytes[header_start..data_start]) + .map_err(|err| format!("invalid NumPy header in {}: {err}", path.display()))?; + if !(header.contains("'descr': ' impl FnOnce(ort::Error) -> String { + move |err| format!("{context}: {err}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shape_len_supports_empty_state_dimensions() { + assert_eq!(shape_len(&[1, 128, 0]).expect("shape"), 0); + assert_eq!(shape_len(&[2, 1, 8, 1000, 64]).expect("shape"), 1_024_000); + } + + #[test] + fn normal_noise_has_requested_length() { + let mut rng = rand::rng(); + assert_eq!(normal_noise(&mut rng, 1, 1.0).len(), 1); + assert_eq!(normal_noise(&mut rng, 32, 1.0).len(), 32); + } + + #[test] + fn generation_frame_estimate_scales_with_token_count() { + assert_eq!(estimate_max_frames(3, 12.5), 38); + assert_eq!(estimate_max_frames(300, 12.5), 1_275); + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"] + fn tokenizer_matches_sentencepiece_reference_including_unknown_words() { + let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle"); + let tokenizer = + load_tokenizer(&Path::new(&dir).join("tokenizer.model")).expect("load April tokenizer"); + let cases: &[(&str, &[u32])] = &[ + ("Yep.", &[2462, 263]), + ("Hello there.", &[2994, 310, 263]), + ( + "quizzaciously xyzzy.", + &[ + 260, 1157, 1818, 362, 1814, 323, 260, 568, 327, 1818, 327, 263, + ], + ), + ("I'm listening.", &[268, 264, 283, 260, 604, 273, 263]), + ]; + for (text, expected) in cases { + let encoding = tokenizer.encode(*text, false).expect("tokenize"); + assert_eq!(encoding.get_ids(), *expected, "{text}"); + } + } + + #[test] + #[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"] + fn loader_splits_oversized_prompts_at_bundle_token_limit() { + let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR") + .expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle"); + let engine = AprilPocketTts::load(Path::new(&dir), PocketPrecision::Fp32, 1) + .expect("load April bundle"); + let text = "This deliberately long sentence repeats ordinary English words so the exact SentencePiece token limit is exercised without relying on punctuation, and it keeps adding more material until the prompt must be divided into multiple independently safe generation chunks before the recurrent state cache can be exhausted."; + let prepared = prepare_april_prompt(text).expect("prepare prompt"); + let chunks = engine.split_prompt(&prepared).expect("split prompt"); + + assert!(chunks.len() > 1); + assert!(chunks.iter().all(|chunk| { + engine.token_count(chunk).expect("tokenize chunk") <= engine.bundle.max_token_per_chunk + })); + } +} diff --git a/crates/buzz-voice/src/pocket_models.rs b/crates/buzz-voice/src/pocket_models.rs new file mode 100644 index 0000000000..ae7ba1a295 --- /dev/null +++ b/crates/buzz-voice/src/pocket_models.rs @@ -0,0 +1,231 @@ +//! Immutable Pocket TTS model capabilities. +//! +//! This module describes caller-provided model files. Download URLs, cache +//! layout, installation, fallback policy, and user-facing selection remain the +//! responsibility of each application. + +/// Pinned upstream export repository for the April model. +pub const APRIL_MODEL_ID: &str = "KevinAHM/pocket-tts-onnx"; + +/// Pinned upstream revision containing the `english_2026-04` bundle. +pub const APRIL_MODEL_REVISION: &str = "58a6d00cf13d239b6748cb0769f35c580a8f606c"; + +/// Language bundle selected from the pinned export. +pub const APRIL_BUNDLE_ID: &str = "english_2026-04"; + +/// Maximum prompt size declared by the April bundle. +pub const APRIL_MAX_TOKEN_PER_CHUNK: usize = 50; + +/// ONNX precision supported by the April runtime. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PocketPrecision { + /// All five selected ONNX graphs use full-precision weights. + Fp32 = 0, + /// FlowLM main, FlowLM flow, and Mimi decoder use upstream INT8 graphs. + /// + /// The pinned upstream runtime keeps Mimi encoder and text conditioner in + /// full precision for this mode. + Int8 = 1, +} + +/// Model selection for a Pocket TTS engine. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum PocketModelSelection { + /// Detect April from `bundle.json`; otherwise retain the January loader. + #[default] + Auto, + /// Load the pinned April bundle with an explicit precision. + English2026_04(PocketPrecision), +} + +/// Runtime configuration independent of application acquisition policy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PocketLoadOptions { + /// Model generation and precision to load. + pub model: PocketModelSelection, + /// ONNX Runtime intra-op thread count. + pub num_threads: usize, +} + +impl Default for PocketLoadOptions { + fn default() -> Self { + Self { + model: PocketModelSelection::Auto, + // A conservative default avoids imposing a desktop-tuned policy on + // mobile callers. Applications may opt into a measured value. + num_threads: 1, + } + } +} + +/// One immutable artifact required by a model precision. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PocketModelArtifact { + pub filename: &'static str, + pub sha256: &'static str, + pub size_bytes: u64, + pub quantized: bool, +} + +/// Capabilities of the effective runtime model. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PocketModelInfo { + pub bundle_id: &'static str, + pub source_model_id: &'static str, + pub revision: Option<&'static str>, + pub precision: PocketPrecision, + pub sample_rate: u32, + pub max_token_per_chunk: Option, + pub artifacts: &'static [PocketModelArtifact], + pub quantized_components: &'static [&'static str], +} + +const COMMON_ARTIFACTS: [PocketModelArtifact; 3] = [ + PocketModelArtifact { + filename: "bundle.json", + sha256: "bab643150f437f37df080a710520ff39ed9ebd9a339f8ebdc739f7eddfc28b3f", + size_bytes: 24_381, + quantized: false, + }, + PocketModelArtifact { + filename: "bos_before_voice.npy", + sha256: "f46edf4f7007b7ba4ea58831f49d003e59e167b4641c44bb3addfe9231a780b1", + size_bytes: 4_224, + quantized: false, + }, + PocketModelArtifact { + filename: "tokenizer.model", + sha256: "d461765ae179566678c93091c5fa6f2984c31bbe990bf1aa62d92c64d91bc3f6", + size_bytes: 59_339, + quantized: false, + }, +]; + +const FP32_ARTIFACTS: [PocketModelArtifact; 8] = [ + COMMON_ARTIFACTS[0], + COMMON_ARTIFACTS[1], + COMMON_ARTIFACTS[2], + PocketModelArtifact { + filename: "flow_lm_main.onnx", + sha256: "6d18315e2c33ca3e3aa4a4e3dca22f56d007fd823127e24948b37695bf54190f", + size_bytes: 302_742_149, + quantized: false, + }, + PocketModelArtifact { + filename: "flow_lm_flow.onnx", + sha256: "085d239f68897e28fb06e95c743738ad8b8c092ee6dc55f5491313e81ff08062", + size_bytes: 39_097_095, + quantized: false, + }, + PocketModelArtifact { + filename: "mimi_decoder.onnx", + sha256: "86f038caa02a96a0ff9c25526a0ff43a4906c418197ed72d3e30f720ac7ce802", + size_bytes: 41_471_926, + quantized: false, + }, + PocketModelArtifact { + filename: "mimi_encoder.onnx", + sha256: "853e2ca623b8782d94c3745ec6133bfdff7ce33d9b11128bd29ea03f28d76e3d", + size_bytes: 39_768_446, + quantized: false, + }, + PocketModelArtifact { + filename: "text_conditioner.onnx", + sha256: "4ecee995fb69f85c7a7493d11f7b5ee15d9950facc7ab3f5c9c49ef1e03847bb", + size_bytes: 16_388_344, + quantized: false, + }, +]; + +const INT8_ARTIFACTS: [PocketModelArtifact; 8] = [ + COMMON_ARTIFACTS[0], + COMMON_ARTIFACTS[1], + COMMON_ARTIFACTS[2], + PocketModelArtifact { + filename: "flow_lm_main_int8.onnx", + sha256: "f9bd8106b79a0192c1c43399ab938fb24900a95c1c599870d75a884e99000116", + size_bytes: 76_341_079, + quantized: true, + }, + PocketModelArtifact { + filename: "flow_lm_flow_int8.onnx", + sha256: "3dd781ee5abee9e195320bf0106bebd6372a852b3b36352524ee78b40554635d", + size_bytes: 9_962_530, + quantized: true, + }, + PocketModelArtifact { + filename: "mimi_decoder_int8.onnx", + sha256: "3630450a3297a101792a6ac66619ebc70ab916b265e6220c2afaef8b1673f925", + size_bytes: 22_684_077, + quantized: true, + }, + // The pinned upstream precision selector intentionally retains these two + // full-precision graphs even though additional INT8 exports exist. + FP32_ARTIFACTS[6], + FP32_ARTIFACTS[7], +]; + +const INT8_COMPONENTS: [&str; 3] = ["flow_lm_main", "flow_lm_flow", "mimi_decoder"]; + +/// Return immutable metadata for an April precision. +pub const fn april_model_info(precision: PocketPrecision) -> PocketModelInfo { + let (artifacts, quantized_components): ( + &'static [PocketModelArtifact], + &'static [&'static str], + ) = match precision { + PocketPrecision::Fp32 => (&FP32_ARTIFACTS, &[]), + PocketPrecision::Int8 => (&INT8_ARTIFACTS, &INT8_COMPONENTS), + }; + + PocketModelInfo { + bundle_id: APRIL_BUNDLE_ID, + source_model_id: APRIL_MODEL_ID, + revision: Some(APRIL_MODEL_REVISION), + precision, + sample_rate: 24_000, + max_token_per_chunk: Some(APRIL_MAX_TOKEN_PER_CHUNK), + artifacts, + quantized_components, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn precision_metadata_matches_pinned_upstream_layout() { + let fp32 = april_model_info(PocketPrecision::Fp32); + let int8 = april_model_info(PocketPrecision::Int8); + + assert_eq!(fp32.artifacts.len(), 8); + assert_eq!(int8.artifacts.len(), 8); + assert_eq!( + int8.quantized_components, + ["flow_lm_main", "flow_lm_flow", "mimi_decoder"] + ); + assert!(int8 + .artifacts + .iter() + .any(|artifact| artifact.filename == "mimi_encoder.onnx" && !artifact.quantized)); + assert!(!int8 + .artifacts + .iter() + .any(|artifact| artifact.filename == "mimi_encoder_int8.onnx")); + assert_eq!( + fp32.artifacts + .iter() + .map(|artifact| artifact.size_bytes) + .sum::(), + 439_555_904 + ); + assert_eq!( + int8.artifacts + .iter() + .map(|artifact| artifact.size_bytes) + .sum::(), + 165_232_420 + ); + } +} diff --git a/crates/buzz-voice/src/preparation.rs b/crates/buzz-voice/src/preparation.rs new file mode 100644 index 0000000000..5661a1aad3 --- /dev/null +++ b/crates/buzz-voice/src/preparation.rs @@ -0,0 +1,784 @@ +//! Shared text preparation and PCM shaping for TTS output. +//! +//! Mental model: +//! +//! ```text +//! raw agent text +//! → strip fenced code blocks → "code block omitted" +//! → strip inline code → bare text +//! → strip URLs → "link omitted" +//! → strip markdown markers → plain text +//! → strip emoji → (removed) +//! → numbers → words → "forty two" +//! → collapse whitespace → clean string +//! ``` +//! +//! Also provides `split_sentences` — the single sentence-boundary splitter used +//! by both the TTS batching pipeline and the Supertonic text chunker. + +use regex::Regex; +use std::sync::LazyLock; + +// ── Sentence splitting ──────────────────────────────────────────────────────── + +/// Regex: a sentence-ending punctuation mark followed by whitespace. +static RE_SENTENCE_BOUNDARY: LazyLock = LazyLock::new(|| Regex::new(r"([.!?])\s+").unwrap()); + +/// Common abbreviations that end with a period but are NOT sentence boundaries. +const ABBREVIATIONS: &[&str] = &[ + "Dr.", "Mr.", "Mrs.", "Ms.", "Prof.", "Sr.", "Jr.", "St.", "Ave.", "Rd.", "Blvd.", "Dept.", + "Inc.", "Ltd.", "Co.", "Corp.", "etc.", "vs.", "i.e.", "e.g.", "Ph.D.", +]; + +/// Split text into sentence-sized chunks. +/// +/// Combines regex-based boundary detection with: +/// - Abbreviation awareness (`Dr.`, `Mr.`, etc. don't split) +/// - Digit-before-period check (avoids splitting `1.` `2.` numbered lists) +/// - `\n` and `—` treated as sentence breaks +/// +/// Returns non-empty, trimmed strings. +pub fn split_sentences(text: &str) -> Vec { + // First, split on newlines and em-dashes to get coarse segments. + let coarse: Vec<&str> = text.split(['\n', '—']).collect(); + + let mut sentences = Vec::new(); + + for segment in coarse { + let segment = segment.trim(); + if segment.is_empty() { + continue; + } + // Within each segment, split on sentence-ending punctuation. + let matches: Vec<_> = RE_SENTENCE_BOUNDARY.find_iter(segment).collect(); + if matches.is_empty() { + sentences.push(segment.to_string()); + continue; + } + + let mut last_end = 0usize; + for m in &matches { + let before = &segment[last_end..m.start()]; + let punc_char = &segment[m.start()..m.start() + 1]; + + // Skip if this looks like an abbreviation. + let combined = format!("{}{}", before.trim(), punc_char); + let is_abbrev = ABBREVIATIONS.iter().any(|a| combined.ends_with(a)); + + // Skip if the character before the period is a digit (numbered list). + let is_digit_period = punc_char == "." + && !before.is_empty() + && before.ends_with(|c: char| c.is_ascii_digit()); + + if !is_abbrev && !is_digit_period { + let piece = segment[last_end..m.end()].trim(); + if !piece.is_empty() { + sentences.push(piece.to_string()); + } + last_end = m.end(); + } + } + + if last_end < segment.len() { + let tail = segment[last_end..].trim(); + if !tail.is_empty() { + sentences.push(tail.to_string()); + } + } + } + + if sentences.is_empty() { + vec![text.to_string()] + } else { + sentences + } +} + +// ── Public API ──────────────────────────────────────────────────────────────── + +/// Prepare `text` for TTS synthesis. +/// +/// Applies in order: +/// 1. Fenced code blocks → "code block omitted" +/// 2. Inline code → bare content (backticks stripped) +/// 3. URLs → "link omitted" +/// 4. Markdown bold/italic/underline markers stripped +/// 5. Emoji stripped +/// 6. Numbers → words (integers 0–999, times HH:MM) +/// 7. Excess whitespace collapsed +pub fn preprocess_for_tts(text: &str) -> String { + let s = strip_fenced_code_blocks(text); + let s = strip_inline_code(&s); + let s = strip_urls(&s); + let s = strip_markdown_markers(&s); + let s = strip_emoji(&s); + let s = expand_numbers(&s); + let s = collapse_whitespace(&s); + // Filter trivially short results — ".", ",", etc. would be spoken as + // "period", "comma" by TTS. Agents that have nothing relevant to say + // should not respond at all, but defense-in-depth catches edge cases. + if s.len() <= 1 { + return String::new(); + } + s +} + +// ── Step implementations ────────────────────────────────────────────────────── + +/// Replace fenced code blocks with "code block omitted". +/// +/// Handles both ` ``` ` and `~~~` fences. Multi-line aware. +fn strip_fenced_code_blocks(text: &str) -> String { + let s = replace_fenced(text, "```"); + replace_fenced(&s, "~~~") +} + +fn replace_fenced(text: &str, fence: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + match rest.find(fence) { + None => { + out.push_str(rest); + break; + } + Some(start) => { + // Everything before the opening fence. + out.push_str(&rest[..start]); + rest = &rest[start + fence.len()..]; + // Skip optional language tag on the same line. + if let Some(nl) = rest.find('\n') { + rest = &rest[nl + 1..]; + } + // Find the closing fence. + match rest.find(fence) { + None => { + // Unclosed fence — treat rest as omitted. + out.push_str(" code block omitted "); + break; + } + Some(end) => { + out.push_str(" code block omitted "); + rest = &rest[end + fence.len()..]; + // Skip trailing newline after closing fence. + if rest.starts_with('\n') { + rest = &rest[1..]; + } + } + } + } + } + } + out +} + +/// Strip backtick-delimited inline code, leaving the inner text. +/// +/// Single-backtick only — triple backtick already handled above. +fn strip_inline_code(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + match rest.find('`') { + None => { + out.push_str(rest); + break; + } + Some(start) => { + out.push_str(&rest[..start]); + rest = &rest[start + 1..]; + match rest.find('`') { + None => { + // Unclosed — emit as-is. + out.push_str(rest); + break; + } + Some(end) => { + out.push_str(&rest[..end]); + rest = &rest[end + 1..]; + } + } + } + } + } + out +} + +/// Replace http/https URLs with "link omitted". +/// +/// Trailing sentence-ending punctuation (`.`, `!`, `?`) that immediately follows +/// a URL and is at end-of-string or followed by whitespace is preserved so that +/// sentence splitting and TTS prosody are not degraded. +/// +/// Example: `"See https://x.y/z."` → `"See link omitted."` +fn strip_urls(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + // Find the earliest URL prefix. + let http = rest.find("http://"); + let https = rest.find("https://"); + let url_start = match (http, https) { + (None, None) => { + out.push_str(rest); + break; + } + (Some(a), None) => a, + (None, Some(b)) => b, + (Some(a), Some(b)) => a.min(b), + }; + out.push_str(&rest[..url_start]); + rest = &rest[url_start..]; + // Consume until whitespace or structural delimiter. + let url_end = rest + .find(|c: char| c.is_whitespace() || c == ')' || c == ']' || c == '"' || c == '\'') + .unwrap_or(rest.len()); + let url_token = &rest[..url_end]; + rest = &rest[url_end..]; + + // Check if the URL token ends with sentence-ending punctuation that + // belongs to the surrounding sentence rather than the URL itself. + // A trailing `.`, `!`, or `?` is preserved when it is at end-of-string + // or followed by whitespace (i.e. it is a sentence boundary). + let trailing_punct = if url_token.ends_with(['.', '!', '?']) { + let after = rest; // rest is already past url_end + if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { + // Preserve the trailing punctuation. + &url_token[url_token.len() - 1..] + } else { + "" + } + } else { + "" + }; + + out.push_str("link omitted"); + out.push_str(trailing_punct); + } + out +} + +/// Strip `**`, `*`, `__`, `_emphasis_`, `~~` markdown markers. +/// +/// Underscores are only stripped when they wrap a word (`_text_`). +/// Standalone underscores (e.g. `snake_case` identifiers) are preserved. +fn strip_markdown_markers(text: &str) -> String { + // Order matters: strip multi-char markers before single-char. + let s = text.replace("**", ""); + let s = s.replace("__", ""); + let s = s.replace("~~", ""); + let s = s.replace('*', ""); + strip_underscore_emphasis(&s) +} + +/// Strip `_text_` emphasis markers while preserving underscores in identifiers. +/// +/// A `_` is treated as an emphasis delimiter only when it is preceded by +/// whitespace or the start of the string AND followed by a non-whitespace char, +/// or vice-versa for the closing delimiter. +fn strip_underscore_emphasis(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let chars: Vec = text.chars().collect(); + let len = chars.len(); + let mut i = 0; + while i < len { + if chars[i] == '_' { + // Opening delimiter: preceded by whitespace/start, followed by non-whitespace. + let prev_is_boundary = i == 0 || chars[i - 1].is_whitespace(); + let next_is_nonspace = i + 1 < len && !chars[i + 1].is_whitespace(); + if prev_is_boundary && next_is_nonspace { + // Look for a matching closing `_`. + if let Some(close) = (i + 1..len).find(|&j| { + chars[j] == '_' + && !chars[j - 1].is_whitespace() + && (j + 1 >= len + || chars[j + 1].is_whitespace() + || chars[j + 1].is_ascii_punctuation()) + }) { + // Emit the inner text without the delimiters. + for &ch in &chars[i + 1..close] { + out.push(ch); + } + i = close + 1; + continue; + } + } + // Not an emphasis delimiter — emit as-is. + out.push('_'); + } else { + out.push(chars[i]); + } + i += 1; + } + out +} + +/// Strip Unicode emoji (characters in common emoji ranges). +/// +/// Covers the main Emoji block (U+1F300–U+1FAFF) and supplemental ranges. +/// ASCII emoticons like `:)` are left as-is. +fn strip_emoji(text: &str) -> String { + text.chars().filter(|&c| !is_emoji(c)).collect() +} + +#[inline] +fn is_emoji(c: char) -> bool { + matches!(c, + '\u{1F300}'..='\u{1FAFF}' // Misc symbols, emoticons, transport, etc. + | '\u{2600}'..='\u{27BF}' // Misc symbols, dingbats + | '\u{FE00}'..='\u{FE0F}' // Variation selectors + | '\u{1F000}'..='\u{1F02F}'// Mahjong/domino tiles + | '\u{1F0A0}'..='\u{1F0FF}'// Playing cards + | '\u{200D}' // Zero-width joiner (used in emoji sequences) + | '\u{20E3}' // Combining enclosing keycap + ) +} + +/// Expand numbers to spoken words. +/// +/// Handles: +/// - Times: `HH:MM` → "eleven thirty" +/// - Integers 0–999,999 +/// - Leaves other numeric strings (e.g. "3.14", "1000000+") as-is. +fn expand_numbers(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.char_indices().peekable(); + + while let Some((i, c)) = chars.next() { + if c.is_ascii_digit() { + // Collect the full token (digits, colon, dots). + let start = i; + let mut end = i + c.len_utf8(); + while let Some(&(j, nc)) = chars.peek() { + if nc.is_ascii_digit() || nc == ':' || nc == '.' { + end = j + nc.len_utf8(); + chars.next(); + } else { + break; + } + } + let token = &text[start..end]; + out.push_str(&expand_numeric_token(token)); + } else { + out.push(c); + } + } + out +} + +fn expand_numeric_token(token: &str) -> String { + // Strip trailing punctuation that the token collector may have included + // (e.g. "11:30." from "at 11:30.") before attempting to parse. + let token = token.trim_end_matches(|c: char| !c.is_ascii_digit()); + + // Time: HH:MM + if let Some(colon) = token.find(':') { + let h = &token[..colon]; + let m = &token[colon + 1..]; + if let (Ok(hh), Ok(mm)) = (h.parse::(), m.parse::()) { + if hh < 24 && mm < 60 { + let hour_word = int_to_words(hh); + let min_word = if mm == 0 { + String::new() + } else if mm < 10 { + // "9:05" → "nine oh five" (not "nine five") + format!(" oh {}", int_to_words(mm)) + } else { + format!(" {}", int_to_words(mm)) + }; + return format!("{}{}", hour_word, min_word); + } + } + // Not a valid time — return as-is. + return token.to_string(); + } + + // Plain integer 0–999,999. + if token.chars().all(|c| c.is_ascii_digit()) { + if let Ok(n) = token.parse::() { + if n <= 999_999 { + return int_to_words(n); + } + } + } + + // Anything else (decimals, millions+) — leave as-is. + token.to_string() +} + +/// Convert an integer 0–999,999 to English words. +fn int_to_words(n: u32) -> String { + const ONES: &[&str] = &[ + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", + "ten", + "eleven", + "twelve", + "thirteen", + "fourteen", + "fifteen", + "sixteen", + "seventeen", + "eighteen", + "nineteen", + ]; + const TENS: &[&str] = &[ + "", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety", + ]; + + if n < 20 { + return ONES[n as usize].to_string(); + } + if n < 100 { + let ten = TENS[(n / 10) as usize]; + let one = n % 10; + return if one == 0 { + ten.to_string() + } else { + format!("{} {}", ten, ONES[one as usize]) + }; + } + if n < 1000 { + let hundreds = n / 100; + let remainder = n % 100; + let hundred_word = format!("{} hundred", ONES[hundreds as usize]); + return if remainder == 0 { + hundred_word + } else { + format!("{} {}", hundred_word, int_to_words(remainder)) + }; + } + // 1,000–999,999 + let thousands = n / 1000; + let remainder = n % 1000; + let thousand_word = format!("{} thousand", int_to_words(thousands)); + if remainder == 0 { + thousand_word + } else { + format!("{} {}", thousand_word, int_to_words(remainder)) + } +} + +/// Collapse runs of whitespace (spaces, tabs, newlines) to a single space. +/// Trims leading/trailing whitespace. +fn collapse_whitespace(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut prev_space = true; // Start true to trim leading whitespace. + for c in text.chars() { + if c.is_whitespace() { + if !prev_space { + out.push(' '); + prev_space = true; + } + } else { + out.push(c); + prev_space = false; + } + } + // Trim trailing space. + if out.ends_with(' ') { + out.pop(); + } + out +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +/// Character budget for greedy synthesis chunks after the latency-critical +/// first sentence. +pub const TTS_CHUNK_MAX_CHARS: usize = 200; + +/// Pocket's output sample rate. +pub const TTS_SAMPLE_RATE: usize = 24_000; +const FADE_OUT_SAMPLES: usize = TTS_SAMPLE_RATE * 8 / 1_000; +const LEAD_IN_SAMPLES: usize = TTS_SAMPLE_RATE * 20 / 1_000; +const TRAILING_SILENCE_SAMPLES: usize = TTS_SAMPLE_RATE * 80 / 1_000; + +/// Convert submitted assistant text into desktop-compatible synthesis chunks. +/// +/// The first sentence stands alone to minimize time to first playback. +/// Remaining sentences are packed greedily up to 200 characters. +pub fn prepare_tts_chunks(raw: &str) -> Vec { + let text = preprocess_for_tts(raw); + if text.is_empty() { + return Vec::new(); + } + let sentences: Vec = split_sentences(&text) + .into_iter() + .filter(|sentence| !sentence.trim().is_empty()) + .collect(); + group_sentences_into_chunks(&sentences, TTS_CHUNK_MAX_CHARS) +} + +/// Group already-prepared sentences using Pocket's desktop chunking policy. +pub fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec { + let mut chunks: Vec = Vec::new(); + for (index, sentence) in sentences.iter().enumerate() { + let sentence = sentence.trim(); + if sentence.is_empty() { + continue; + } + if index == 0 || chunks.is_empty() { + chunks.push(sentence.to_owned()); + continue; + } + let can_merge = chunks.len() > 1 + && chunks + .last() + .is_some_and(|chunk| chunk.len() + 1 + sentence.len() <= max_chars); + if can_merge { + if let Some(chunk) = chunks.last_mut() { + chunk.push(' '); + chunk.push_str(sentence); + } + } else { + chunks.push(sentence.to_owned()); + } + } + chunks +} + +/// Clamp and shape a Pocket chunk for glitch-free playback. +/// +/// Applies the desktop 8 ms fade-out, then prepends 20 ms and appends 80 ms +/// of silence, preserving the established 100 ms inter-chunk gap. +pub fn shape_tts_chunk(samples: Vec) -> Vec { + let mut audio: Vec = samples + .into_iter() + .map(|sample| sample.clamp(-1.0, 1.0)) + .collect(); + let fade = FADE_OUT_SAMPLES.min(audio.len() / 2); + for index in 0..fade { + let sample_index = audio.len() - 1 - index; + audio[sample_index] *= index as f32 / fade as f32; + } + + let mut shaped = Vec::with_capacity(LEAD_IN_SAMPLES + audio.len() + TRAILING_SILENCE_SAMPLES); + shaped.resize(LEAD_IN_SAMPLES, 0.0); + shaped.extend(audio); + shaped.resize(shaped.len() + TRAILING_SILENCE_SAMPLES, 0.0); + shaped +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_fenced_code_block() { + let input = "Here is some code:\n```rust\nfn main() {}\n```\nDone."; + let out = preprocess_for_tts(input); + assert!(out.contains("code block omitted"), "got: {out}"); + assert!(!out.contains("fn main"), "got: {out}"); + } + + #[test] + fn strips_inline_code() { + let out = preprocess_for_tts("Call `foo()` now."); + assert_eq!(out, "Call foo() now."); + } + + #[test] + fn strips_urls() { + let out = preprocess_for_tts("See https://example.com for details."); + assert!(out.contains("link omitted"), "got: {out}"); + assert!(!out.contains("example.com"), "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_period() { + // Trailing `.` at end of sentence must be preserved for sentence splitting. + let out = strip_urls("See https://x.y/z."); + assert_eq!(out, "See link omitted.", "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_exclamation() { + let out = strip_urls("Visit https://example.com!"); + assert_eq!(out, "Visit link omitted!", "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_question() { + let out = strip_urls("Did you see https://example.com?"); + assert_eq!(out, "Did you see link omitted?", "got: {out}"); + } + + #[test] + fn strips_url_mid_sentence_no_punct_preserved() { + // URL in the middle of a sentence — no trailing punct to preserve. + let out = strip_urls("Check https://example.com for more info."); + assert_eq!(out, "Check link omitted for more info.", "got: {out}"); + } + + #[test] + fn strips_bold_italic() { + let out = preprocess_for_tts("**bold** and *italic* and _under_"); + assert_eq!(out, "bold and italic and under"); + } + + #[test] + fn preserves_standalone_underscores() { + // snake_case identifiers should not be mangled. + let out = preprocess_for_tts("call foo_bar() or baz_qux"); + assert!(out.contains("foo_bar"), "got: {out}"); + assert!(out.contains("baz_qux"), "got: {out}"); + } + + #[test] + fn strips_tilde_fenced_block() { + let input = "Here:\n~~~python\nprint('hi')\n~~~\nDone."; + let out = preprocess_for_tts(input); + assert!(out.contains("code block omitted"), "got: {out}"); + assert!(!out.contains("print"), "got: {out}"); + } + + #[test] + fn expands_integers() { + assert_eq!(preprocess_for_tts("42"), "forty two"); + assert_eq!(preprocess_for_tts("0"), "zero"); + assert_eq!(preprocess_for_tts("11"), "eleven"); + assert_eq!(preprocess_for_tts("100"), "one hundred"); + } + + #[test] + fn expands_thousands() { + assert_eq!(preprocess_for_tts("1000"), "one thousand"); + assert_eq!( + preprocess_for_tts("1234"), + "one thousand two hundred thirty four" + ); + assert_eq!(preprocess_for_tts("10000"), "ten thousand"); + assert_eq!(preprocess_for_tts("100000"), "one hundred thousand"); + assert_eq!( + preprocess_for_tts("999999"), + "nine hundred ninety nine thousand nine hundred ninety nine" + ); + } + + #[test] + fn expands_times() { + assert_eq!(preprocess_for_tts("11:30"), "eleven thirty"); + assert_eq!(preprocess_for_tts("9:00"), "nine"); + assert_eq!(preprocess_for_tts("9:05"), "nine oh five"); + assert_eq!(preprocess_for_tts("10:09"), "ten oh nine"); + } + + #[test] + fn collapses_whitespace() { + let out = preprocess_for_tts(" hello world "); + assert_eq!(out, "hello world"); + } + + #[test] + fn split_sentences_basic() { + let result = split_sentences("Hello world. How are you? I'm fine!"); + assert_eq!(result, vec!["Hello world.", "How are you?", "I'm fine!"]); + } + + #[test] + fn split_sentences_newline_break() { + let result = split_sentences("First line.\nSecond line."); + assert_eq!(result, vec!["First line.", "Second line."]); + } + + #[test] + fn split_sentences_em_dash_break() { + let result = split_sentences("Start here—then continue."); + assert_eq!(result, vec!["Start here", "then continue."]); + } + + #[test] + fn split_sentences_abbreviations() { + let result = split_sentences("Dr. Smith went home. He was tired."); + assert_eq!(result, vec!["Dr. Smith went home.", "He was tired."]); + } + + #[test] + fn split_sentences_numbered_list() { + let result = split_sentences("1. First item. 2. Second item."); + // "1." and "2." should NOT cause a split (digit before period). + assert_eq!(result, vec!["1. First item.", "2. Second item."]); + } + + #[test] + fn split_sentences_single() { + let result = split_sentences("Just one sentence"); + assert_eq!(result, vec!["Just one sentence"]); + } + + #[test] + fn split_sentences_empty() { + let result = split_sentences(""); + assert_eq!(result, vec![""]); + } + + #[test] + fn filters_trivial_responses() { + assert_eq!(preprocess_for_tts("."), ""); + assert_eq!(preprocess_for_tts(","), ""); + assert_eq!(preprocess_for_tts("!"), ""); + assert_eq!(preprocess_for_tts(" "), ""); + assert_eq!(preprocess_for_tts("ok"), "ok"); + } + + #[test] + fn full_pipeline() { + let input = + "**Agent says:** check https://relay.example.com at 11:30.\n```\nsome code\n```"; + let out = preprocess_for_tts(input); + assert!(!out.contains("**"), "got: {out}"); + assert!(!out.contains("https://"), "got: {out}"); + assert!(out.contains("eleven thirty"), "got: {out}"); + assert!(out.contains("code block omitted"), "got: {out}"); + } + + #[test] + fn preparation_keeps_first_sentence_latency_chunk_separate() { + assert_eq!( + prepare_tts_chunks("First. Second. Third."), + vec!["First.", "Second. Third."] + ); + } + + #[test] + fn shaping_matches_desktop_padding_clamp_and_fade() { + let shaped = shape_tts_chunk(vec![2.0; 400]); + assert_eq!(shaped.len(), 480 + 400 + 1920); + assert!(shaped[..480].iter().all(|sample| *sample == 0.0)); + assert_eq!(shaped[480], 1.0); + assert_eq!(shaped[879], 0.0); + assert!(shaped[880..].iter().all(|sample| *sample == 0.0)); + } + + #[test] + fn shaping_preserves_the_leading_audio_samples() { + let input: Vec = (0..FADE_OUT_SAMPLES * 4) + .map(|index| 0.5 + index as f32 * 1e-4) + .collect(); + let shaped = shape_tts_chunk(input.clone()); + assert_eq!( + &shaped[LEAD_IN_SAMPLES..LEAD_IN_SAMPLES + FADE_OUT_SAMPLES], + &input[..FADE_OUT_SAMPLES] + ); + } + + #[test] + fn shaping_short_and_empty_audio_remains_well_formed() { + let single = shape_tts_chunk(vec![0.5]); + assert_eq!(single[LEAD_IN_SAMPLES], 0.5); + assert_eq!(single.len(), LEAD_IN_SAMPLES + 1 + TRAILING_SILENCE_SAMPLES); + + let empty = shape_tts_chunk(Vec::new()); + assert_eq!(empty.len(), LEAD_IN_SAMPLES + TRAILING_SILENCE_SAMPLES); + assert!(empty.iter().all(|sample| *sample == 0.0)); + } +} diff --git a/crates/sherpa-onnx-sys/Cargo.toml b/crates/sherpa-onnx-sys/Cargo.toml new file mode 100644 index 0000000000..317d2c5b26 --- /dev/null +++ b/crates/sherpa-onnx-sys/Cargo.toml @@ -0,0 +1,63 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "sherpa-onnx-sys" +version = "1.13.4" +build = "build.rs" +links = "sherpa-onnx" +include = [ + "src/**", + "build.rs", + "Cargo.toml", + "README.md", + "LICENSE*", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Raw FFI bindings to the sherpa-onnx C API" +readme = "README.md" +keywords = [ + "ffi", + "speech", + "sherpa-onnx", + "bindings", +] +categories = ["external-ffi-bindings"] +license = "Apache-2.0" +repository = "https://github.com/k2-fsa/sherpa-onnx" + +[package.metadata.docs.rs] +default-features = false +features = [] + +[features] +default = ["static"] +shared = [] +static = [] + +[lib] +name = "sherpa_onnx_sys" +path = "src/lib.rs" + +[build-dependencies.bzip2] +version = "0.4" + +[build-dependencies.tar] +version = "0.4" + +[build-dependencies.ureq] +version = "2.12" +features = ["proxy-from-env"] diff --git a/crates/sherpa-onnx-sys/Cargo.toml.orig b/crates/sherpa-onnx-sys/Cargo.toml.orig new file mode 100644 index 0000000000..5744d49363 --- /dev/null +++ b/crates/sherpa-onnx-sys/Cargo.toml.orig @@ -0,0 +1,34 @@ +[package] +name = "sherpa-onnx-sys" +version = "1.13.4" +edition = "2021" +description = "Raw FFI bindings to the sherpa-onnx C API" +license = "Apache-2.0" +repository = "https://github.com/k2-fsa/sherpa-onnx" +readme = "README.md" +links = "sherpa-onnx" + +keywords = ["ffi", "speech", "sherpa-onnx", "bindings"] +categories = ["external-ffi-bindings"] + +include = [ + "src/**", + "build.rs", + "Cargo.toml", + "README.md", + "LICENSE*", +] + +[features] +default = ["static"] +static = [] +shared = [] + +[build-dependencies] +bzip2 = "0.4" +tar = "0.4" +ureq = { version = "2.12", features = ["proxy-from-env"] } + +[package.metadata.docs.rs] +default-features = false +features = [] diff --git a/crates/sherpa-onnx-sys/LICENSE b/crates/sherpa-onnx-sys/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/crates/sherpa-onnx-sys/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/sherpa-onnx-sys/README.md b/crates/sherpa-onnx-sys/README.md new file mode 100644 index 0000000000..689fa55575 --- /dev/null +++ b/crates/sherpa-onnx-sys/README.md @@ -0,0 +1,671 @@ +
+ +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/k2-fsa/sherpa-onnx) + +
+ +Buzz carries the crates.io 1.13.4 FFI sources and locally patches the build +script to link caller-supplied official iOS and Android libraries through +`SHERPA_ONNX_LIB_DIR`. Desktop archive selection remains unchanged. + + ### Supported functions + +|Speech recognition| [Speech synthesis][tts-url] | [Source separation][ss-url] | +|------------------|------------------|-------------------| +| ✔️ | ✔️ | ✔️ | + +|Speaker identification| [Speaker diarization][sd-url] | Speaker verification | +|----------------------|-------------------- |------------------------| +| ✔️ | ✔️ | ✔️ | + +| [Spoken Language identification][slid-url] | [Audio tagging][at-url] | [Voice activity detection][vad-url] | +|--------------------------------|---------------|--------------------------| +| ✔️ | ✔️ | ✔️ | + +| [Keyword spotting][kws-url] | [Add punctuation][punct-url] | [Speech enhancement][se-url] | +|------------------|-----------------|--------------------| +| ✔️ | ✔️ | ✔️ | + + +### Supported platforms + +|Architecture| Android | iOS | Windows | macOS | linux | HarmonyOS | +|------------|---------|---------|------------|-------|-------|-----------| +| x64 | ✔️ | | ✔️ | ✔️ | ✔️ | ✔️ | +| x86 | ✔️ | | ✔️ | | | | +| arm64 | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | +| arm32 | ✔️ | | | | ✔️ | ✔️ | +| riscv64 | | | | | ✔️ | | + +### Supported programming languages + +| 1. C++ | 2. C | 3. Python | 4. JavaScript | +|--------|-------|-----------|---------------| +| ✔️ | ✔️ | ✔️ | ✔️ | + +|5. Java | 6. C# | 7. Kotlin | 8. Swift | +|--------|-------|-----------|----------| +| ✔️ | ✔️ | ✔️ | ✔️ | + +| 9. Go | 10. Dart | 11. Rust | 12. Pascal | +|-------|----------|----------|------------| +| ✔️ | ✔️ | ✔️ | ✔️ | + + +It also supports WebAssembly. + +### Supported NPUs + +| [1. Rockchip NPU (RKNN)][rknpu-doc] | [2. Qualcomm NPU (QNN)][qnn-doc] | [3. Ascend NPU][ascend-doc] | +|-------------------------------------|-----------------------------------|-----------------------------| +| ✔️ | ✔️ | ✔️ | + +| [4. Axera NPU][axera-npu] | +|---------------------------| +| ✔️ | + +[Join our discord](https://discord.gg/fJdxzg2VbG) + + +## Introduction + +This repository supports running the following functions **locally** + + - Speech-to-text (i.e., ASR); both streaming and non-streaming are supported + - Text-to-speech (i.e., TTS) + - Speaker diarization + - Speaker identification + - Speaker verification + - Spoken language identification + - Audio tagging + - VAD (e.g., [silero-vad][silero-vad]) + - Speech enhancement (e.g., [gtcrn][gtcrn], [DPDFNet](https://github.com/ceva-ip/DPDFNet)) + - Keyword spotting + - Source separation (e.g., [spleeter][spleeter], [UVR][UVR]) + +on the following platforms and operating systems: + + - x86, ``x86_64``, 32-bit ARM, 64-bit ARM (arm64, aarch64), RISC-V (riscv64), **RK NPU**, **Ascend NPU** + - Linux, macOS, Windows, openKylin + - Android, WearOS + - iOS + - HarmonyOS + - NodeJS + - WebAssembly + - [NVIDIA Jetson Orin NX][NVIDIA Jetson Orin NX] (Support running on both CPU and GPU) + - [NVIDIA Jetson Nano B01][NVIDIA Jetson Nano B01] (Support running on both CPU and GPU) + - [Raspberry Pi][Raspberry Pi] + - [RV1126][RV1126] + - [LicheePi4A][LicheePi4A] + - [VisionFive 2][VisionFive 2] + - [旭日X3派][旭日X3派] + - [爱芯派][爱芯派] + - [RK3588][RK3588] + - [SpacemiT-K1][SpacemiT-K1] + - [SpacemiT-K3][SpacemiT-K3] + - etc + +with the following APIs + + - C++, C, Python, Go, ``C#`` + - Java, Kotlin, JavaScript + - Swift, Rust + - Dart, Object Pascal + +### Links for Huggingface Spaces + +
+You can visit the following Huggingface spaces to try sherpa-onnx without +installing anything. All you need is a browser. + +| Description | URL | 中国镜像 | +|-------------------------------------------------------|-----------------------------------------|----------------------------------------| +| Speaker diarization | [Click me][hf-space-speaker-diarization]| [镜像][hf-space-speaker-diarization-cn]| +| Speech recognition | [Click me][hf-space-asr] | [镜像][hf-space-asr-cn] | +| Speech recognition with [Whisper][Whisper] | [Click me][hf-space-asr-whisper] | [镜像][hf-space-asr-whisper-cn] | +| Speech synthesis | [Click me][hf-space-tts] | [镜像][hf-space-tts-cn] | +| Generate subtitles | [Click me][hf-space-subtitle] | [镜像][hf-space-subtitle-cn] | +| Audio tagging | [Click me][hf-space-audio-tagging] | [镜像][hf-space-audio-tagging-cn] | +| Source separation | [Click me][hf-space-source-separation] | [镜像][hf-space-source-separation-cn] | +| Spoken language identification with [Whisper][Whisper]| [Click me][hf-space-slid-whisper] | [镜像][hf-space-slid-whisper-cn] | + +We also have spaces built using WebAssembly. They are listed below: + +| Description | Huggingface space| ModelScope space| +|------------------------------------------------------------------------------------------|------------------|-----------------| +|Voice activity detection with [silero-vad][silero-vad] | [Click me][wasm-hf-vad]|[地址][wasm-ms-vad]| +|Real-time speech recognition (Chinese + English) with Zipformer | [Click me][wasm-hf-streaming-asr-zh-en-zipformer]|[地址][wasm-hf-streaming-asr-zh-en-zipformer]| +|Real-time speech recognition (Chinese + English) with Paraformer |[Click me][wasm-hf-streaming-asr-zh-en-paraformer]| [地址][wasm-ms-streaming-asr-zh-en-paraformer]| +|Real-time speech recognition (Chinese + English + Cantonese) with [Paraformer-large][Paraformer-large]|[Click me][wasm-hf-streaming-asr-zh-en-yue-paraformer]| [地址][wasm-ms-streaming-asr-zh-en-yue-paraformer]| +|Real-time speech recognition (English) |[Click me][wasm-hf-streaming-asr-en-zipformer] |[地址][wasm-ms-streaming-asr-en-zipformer]| +|VAD + speech recognition (Chinese) with [Zipformer CTC](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/icefall/zipformer.html#sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03-chinese)|[Click me][wasm-hf-vad-asr-zh-zipformer-ctc-07-03]| [地址][wasm-ms-vad-asr-zh-zipformer-ctc-07-03]| +|VAD + speech recognition (Chinese + English + Korean + Japanese + Cantonese) with [SenseVoice][SenseVoice]|[Click me][wasm-hf-vad-asr-zh-en-ko-ja-yue-sense-voice]| [地址][wasm-ms-vad-asr-zh-en-ko-ja-yue-sense-voice]| +|VAD + speech recognition (English) with [Whisper][Whisper] tiny.en|[Click me][wasm-hf-vad-asr-en-whisper-tiny-en]| [地址][wasm-ms-vad-asr-en-whisper-tiny-en]| +|VAD + speech recognition (English) with [Moonshine tiny][Moonshine tiny]|[Click me][wasm-hf-vad-asr-en-moonshine-tiny-en]| [地址][wasm-ms-vad-asr-en-moonshine-tiny-en]| +|VAD + speech recognition (English) with Zipformer trained with [GigaSpeech][GigaSpeech] |[Click me][wasm-hf-vad-asr-en-zipformer-gigaspeech]| [地址][wasm-ms-vad-asr-en-zipformer-gigaspeech]| +|VAD + speech recognition (Chinese) with Zipformer trained with [WenetSpeech][WenetSpeech] |[Click me][wasm-hf-vad-asr-zh-zipformer-wenetspeech]| [地址][wasm-ms-vad-asr-zh-zipformer-wenetspeech]| +|VAD + speech recognition (Japanese) with Zipformer trained with [ReazonSpeech][ReazonSpeech]|[Click me][wasm-hf-vad-asr-ja-zipformer-reazonspeech]| [地址][wasm-ms-vad-asr-ja-zipformer-reazonspeech]| +|VAD + speech recognition (Thai) with Zipformer trained with [GigaSpeech2][GigaSpeech2] |[Click me][wasm-hf-vad-asr-th-zipformer-gigaspeech2]| [地址][wasm-ms-vad-asr-th-zipformer-gigaspeech2]| +|VAD + speech recognition (Chinese 多种方言) with a [TeleSpeech-ASR][TeleSpeech-ASR] CTC model|[Click me][wasm-hf-vad-asr-zh-telespeech]| [地址][wasm-ms-vad-asr-zh-telespeech]| +|VAD + speech recognition (English + Chinese, 及多种中文方言) with Paraformer-large |[Click me][wasm-hf-vad-asr-zh-en-paraformer-large]| [地址][wasm-ms-vad-asr-zh-en-paraformer-large]| +|VAD + speech recognition (English + Chinese, 及多种中文方言) with Paraformer-small |[Click me][wasm-hf-vad-asr-zh-en-paraformer-small]| [地址][wasm-ms-vad-asr-zh-en-paraformer-small]| +|VAD + speech recognition (多语种及多种中文方言) with [Dolphin][Dolphin]-base |[Click me][wasm-hf-vad-asr-multi-lang-dolphin-base]| [地址][wasm-ms-vad-asr-multi-lang-dolphin-base]| +|Speech synthesis (Piper, English) |[Click me][wasm-hf-tts-piper-en]| [地址][wasm-ms-tts-piper-en]| +|Speech synthesis (Piper, German) |[Click me][wasm-hf-tts-piper-de]| [地址][wasm-ms-tts-piper-de]| +|Speech synthesis (Matcha, Chinese) |[Click me][wasm-hf-tts-matcha-zh]| [地址][wasm-ms-tts-matcha-zh]| +|Speech synthesis (Matcha, English) |[Click me][wasm-hf-tts-matcha-en]| [地址][wasm-ms-tts-matcha-en]| +|Speech synthesis (Matcha, Chinese+English) |[Click me][wasm-hf-tts-matcha-zh-en]| [地址][wasm-ms-tts-matcha-zh-en]| +|Speaker diarization |[Click me][wasm-hf-speaker-diarization]|[地址][wasm-ms-speaker-diarization]| +|Voice cloning with ZipVoice (Chinese+English) |[Click me][wasm-hf-voice-cloning-zipvoice]|[地址][wasm-ms-voice-cloning-zipvoice]| +|Voice cloning with Pocket TTS (English) |[Click me][wasm-hf-voice-cloning-pocket]|[地址][wasm-ms-voice-cloning-pocket]| + +
+ +### Links for pre-built Android APKs + +
+ +You can find pre-built Android APKs for this repository in the following table + +| Description | URL | 中国用户 | +|----------------------------------------|------------------------------------|-----------------------------------| +| Speaker diarization | [Address][apk-speaker-diarization] | [点此][apk-speaker-diarization-cn]| +| Streaming speech recognition | [Address][apk-streaming-asr] | [点此][apk-streaming-asr-cn] | +| Simulated-streaming speech recognition | [Address][apk-simula-streaming-asr]| [点此][apk-simula-streaming-asr-cn]| +| Text-to-speech | [Address][apk-tts] | [点此][apk-tts-cn] | +| Voice activity detection (VAD) | [Address][apk-vad] | [点此][apk-vad-cn] | +| VAD + non-streaming speech recognition | [Address][apk-vad-asr] | [点此][apk-vad-asr-cn] | +| Two-pass speech recognition | [Address][apk-2pass] | [点此][apk-2pass-cn] | +| Audio tagging | [Address][apk-at] | [点此][apk-at-cn] | +| Audio tagging (WearOS) | [Address][apk-at-wearos] | [点此][apk-at-wearos-cn] | +| Speaker identification | [Address][apk-sid] | [点此][apk-sid-cn] | +| Spoken language identification | [Address][apk-slid] | [点此][apk-slid-cn] | +| Keyword spotting | [Address][apk-kws] | [点此][apk-kws-cn] | + +
+ +### Links for pre-built Flutter APPs + +
+ +#### Real-time speech recognition + +| Description | URL | 中国用户 | +|--------------------------------|-------------------------------------|-------------------------------------| +| Streaming speech recognition | [Address][apk-flutter-streaming-asr]| [点此][apk-flutter-streaming-asr-cn]| + +#### Text-to-speech + +| Description | URL | 中国用户 | +|------------------------------------------|------------------------------------|------------------------------------| +| Android (arm64-v8a, armeabi-v7a, x86_64) | [Address][flutter-tts-android] | [点此][flutter-tts-android-cn] | +| Linux (x64) | [Address][flutter-tts-linux] | [点此][flutter-tts-linux-cn] | +| macOS (x64) | [Address][flutter-tts-macos-x64] | [点此][flutter-tts-macos-x64-cn] | +| macOS (arm64) | [Address][flutter-tts-macos-arm64] | [点此][flutter-tts-macos-arm64-cn] | +| Windows (x64) | [Address][flutter-tts-win-x64] | [点此][flutter-tts-win-x64-cn] | + +> Note: You need to build from source for iOS. + +
+ +### Links for pre-built Lazarus APPs + +
+ +#### Generating subtitles + +| Description | URL | 中国用户 | +|--------------------------------|----------------------------|----------------------------| +| Generate subtitles (生成字幕) | [Address][lazarus-subtitle]| [点此][lazarus-subtitle-cn]| + +
+ +### Links for pre-trained models + +
+ +| Description | URL | +|---------------------------------------------|---------------------------------------------------------------------------------------| +| Speech recognition (speech to text, ASR) | [Address][asr-models] | +| Text-to-speech (TTS) | [Address][tts-models] | +| VAD | [Address][vad-models] | +| Keyword spotting | [Address][kws-models] | +| Audio tagging | [Address][at-models] | +| Speaker identification (Speaker ID) | [Address][sid-models] | +| Spoken language identification (Language ID)| See multi-lingual [Whisper][Whisper] ASR models from [Speech recognition][asr-models]| +| Punctuation | [Address][punct-models] | +| Speaker segmentation | [Address][speaker-segmentation-models] | +| Speech enhancement | [Address][speech-enhancement-models] | +| Source separation | [Address][source-separation-models] | + +
+ +#### Some pre-trained ASR models (Streaming) + +
+ +Please see + + - + - + - + +for more models. The following table lists only **SOME** of them. + + +|Name | Supported Languages| Description| +|-----|-----|----| +|[sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20][sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20]| Chinese, English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english)| +|[sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16][sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16]| Chinese, English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16-bilingual-chinese-english)| +|[sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23][sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23]|Chinese| Suitable for Cortex A7 CPU. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-zh-14m-2023-02-23)| +|[sherpa-onnx-streaming-zipformer-en-20M-2023-02-17][sherpa-onnx-streaming-zipformer-en-20M-2023-02-17]|English|Suitable for Cortex A7 CPU. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-en-20m-2023-02-17)| +|[sherpa-onnx-streaming-zipformer-korean-2024-06-16][sherpa-onnx-streaming-zipformer-korean-2024-06-16]|Korean| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-korean-2024-06-16-korean)| +|[sherpa-onnx-streaming-zipformer-fr-2023-04-14][sherpa-onnx-streaming-zipformer-fr-2023-04-14]|French| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#shaojieli-sherpa-onnx-streaming-zipformer-fr-2023-04-14-french)| + +
+ + +#### Some pre-trained ASR models (Non-Streaming) + +
+ +Please see + + - + - + - + - + - + +for more models. The following table lists only **SOME** of them. + +|Name | Supported Languages| Description| +|-----|-----|----| +|[sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/nemo-transducer-models.html#sherpa-onnx-nemo-parakeet-tdt-0-6b-v2-int8-english)| English | It is converted from | +|[Whisper tiny.en](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2)|English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html)| +|[Moonshine tiny][Moonshine tiny]|English|See [also](https://github.com/usefulsensors/moonshine)| +|[sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/icefall/zipformer.html#sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03-chinese)|Chinese| A Zipformer CTC model| +|[sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17][sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]|Chinese, Cantonese, English, Korean, Japanese| 支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/sense-voice/index.html)| +|[sherpa-onnx-paraformer-zh-2024-03-09][sherpa-onnx-paraformer-zh-2024-03-09]|Chinese, English| 也支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2024-03-09-chinese-english)| +|[sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01][sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01]|Japanese|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01-japanese)| +|[sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24][sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24]|Russian|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/nemo-transducer-models.html#sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24-russian)| +|[sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24][sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24]|Russian| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/nemo/russian.html#sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24)| +|[sherpa-onnx-zipformer-ru-2024-09-18][sherpa-onnx-zipformer-ru-2024-09-18]|Russian|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-ru-2024-09-18-russian)| +|[sherpa-onnx-zipformer-korean-2024-06-24][sherpa-onnx-zipformer-korean-2024-06-24]|Korean|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-korean-2024-06-24-korean)| +|[sherpa-onnx-zipformer-thai-2024-06-20][sherpa-onnx-zipformer-thai-2024-06-20]|Thai| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-thai-2024-06-20-thai)| +|[sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04][sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04]|Chinese| 支持多种方言. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/telespeech/models.html#sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04)| + +
+ +### Useful links + +- Documentation: https://k2-fsa.github.io/sherpa/onnx/ +- Bilibili 演示视频: https://search.bilibili.com/all?keyword=%E6%96%B0%E4%B8%80%E4%BB%A3Kaldi + +### How to reach us + +Please see +https://k2-fsa.github.io/sherpa/social-groups.html +for 新一代 Kaldi **微信交流群** and **QQ 交流群**. + +## Projects using sherpa-onnx + +### [Sherpa Voice / @siteed/sherpa-onnx.rn](https://github.com/deeeed/audiolab) + +> React Native wrapper and demo app for validating sherpa-onnx on iOS, +> Android, and Web, including ASR, TTS, VAD, KWS, speaker ID, diarization, +> language ID, punctuation, audio tagging, and speech enhancement. + +- [NPM package](https://www.npmjs.com/package/@siteed/sherpa-onnx.rn) +- [Live demo](https://deeeed.github.io/audiolab/sherpa-voice/) + +### [Speed of Sound](https://github.com/zugaldia/speedofsound) + +> A voice-typing application for the Linux desktop (GTK4/Adwaita). +> It captures microphone audio, transcribes it offline using Sherpa ONNX ASR models, +> optionally polishes the text with an LLM, and types the result into the active window +> via XDG Remote Desktop Portal keyboard simulation. + +### [VoxSherpa TTS](https://github.com/CodeBySonu95/VoxSherpa-TTS) + +> VoxSherpa TTS is a 100% offline Android Text-to-Speech app powered by Sherpa-ONNX. +> It supports Kokoro-82M, Piper, and VITS engines with multilingual support including +> Hindi, English, British English, Japanese, Chinese and 50+ more languages. + +- [Download APK (All Versions)](https://github.com/CodeBySonu95/VoxSherpa-TTS/releases) +- Android 11+ · 100% offline · No telemetry + +
+ +| Generate | Models | Library | Settings | +|:---:|:---:|:---:|:---:| +| | | | | + +
+ +--- +### [BreezeApp](https://github.com/mtkresearch/BreezeApp) from [MediaTek Research](https://github.com/mtkresearch) + +> BreezeAPP is a mobile AI application developed for both Android and iOS platforms. +> Users can download it directly from the App Store and enjoy a variety of features +> offline, including speech-to-text, text-to-speech, text-based chatbot interactions, +> and image question-answering + + - [Download APK for BreezeAPP](https://huggingface.co/MediaTek-Research/BreezeApp/resolve/main/BreezeApp.apk) + - [APK 中国镜像](https://hf-mirror.com/MediaTek-Research/BreezeApp/blob/main/BreezeApp.apk) + +| 1 | 2 | 3 | +|---|---|---| +|![](https://github.com/user-attachments/assets/1cdbc057-b893-4de6-9e9c-f1d7dfd1d992)|![](https://github.com/user-attachments/assets/d77cd98e-b057-442f-860d-d5befd5c769b)|![](https://github.com/user-attachments/assets/57e546bf-3d39-45b9-b392-b48ca4fb3c58)| + +### [Open-LLM-VTuber](https://github.com/t41372/Open-LLM-VTuber) + +Talk to any LLM with hands-free voice interaction, voice interruption, and Live2D taking +face running locally across platforms + +See also + +### [voiceapi](https://github.com/ruzhila/voiceapi) + +
+ Streaming ASR and TTS based on FastAPI + + +It shows how to use the ASR and TTS Python APIs with FastAPI. +
+ +### [腾讯会议摸鱼工具 TMSpeech](https://github.com/jxlpzqc/TMSpeech) + +Uses streaming ASR in C# with graphical user interface. + +Video demo in Chinese: [【开源】Windows实时字幕软件（网课/开会必备）](https://www.bilibili.com/video/BV1rX4y1p7Nx) + +### [lol互动助手](https://github.com/l1veIn/lol-wom-electron) + +It uses the JavaScript API of sherpa-onnx along with [Electron](https://electronjs.org/) + +Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率的英雄联盟工具！英雄联盟的最后一块拼图！和游戏中的每个人无障碍沟通！](https://www.bilibili.com/video/BV142tje9E74) + +### [Sherpa-ONNX 语音识别服务器](https://github.com/hfyydd/sherpa-onnx-server) + +A server based on nodejs providing Restful API for speech recognition. + +### [QSmartAssistant](https://github.com/xinhecuican/QSmartAssistant) + +一个模块化，全过程可离线，低占用率的对话机器人/智能音箱 + +It uses QT. Both [ASR](https://github.com/xinhecuican/QSmartAssistant/blob/master/doc/%E5%AE%89%E8%A3%85.md#asr) +and [TTS](https://github.com/xinhecuican/QSmartAssistant/blob/master/doc/%E5%AE%89%E8%A3%85.md#tts) +are used. + +### [Flutter-EasySpeechRecognition](https://github.com/Jason-chen-coder/Flutter-EasySpeechRecognition) + +It extends [./flutter-examples/streaming_asr](./flutter-examples/streaming_asr) by +downloading models inside the app to reduce the size of the app. + +Note: [[Team B] Sherpa AI backend](https://github.com/umgc/spring2025/pull/82) also uses +sherpa-onnx in a Flutter APP. + +### [sherpa-onnx-unity](https://github.com/xue-fei/sherpa-onnx-unity) + +sherpa-onnx in Unity. See also [#1695](https://github.com/k2-fsa/sherpa-onnx/issues/1695), +[#1892](https://github.com/k2-fsa/sherpa-onnx/issues/1892), and [#1859](https://github.com/k2-fsa/sherpa-onnx/issues/1859) + +### [xiaozhi-esp32-server](https://github.com/xinnan-tech/xiaozhi-esp32-server) + +本项目为xiaozhi-esp32提供后端服务，帮助您快速搭建ESP32设备控制服务器 +Backend service for xiaozhi-esp32, helps you quickly build an ESP32 device control server. + +See also + + - [ASR新增轻量级sherpa-onnx-asr](https://github.com/xinnan-tech/xiaozhi-esp32-server/issues/315) + - [feat: ASR增加sherpa-onnx模型](https://github.com/xinnan-tech/xiaozhi-esp32-server/pull/379) + +### [KaithemAutomation](https://github.com/EternityForest/KaithemAutomation) + +Pure Python, GUI-focused home automation/consumer grade SCADA. + +It uses TTS from sherpa-onnx. See also [✨ Speak command that uses the new globally configured TTS model.](https://github.com/EternityForest/KaithemAutomation/commit/8e64d2b138725e426532f7d66bb69dd0b4f53693) + +### [Open-XiaoAI KWS](https://github.com/idootop/open-xiaoai-kws) + +Enable custom wake word for XiaoAi Speakers. 让小爱音箱支持自定义唤醒词。 + +Video demo in Chinese: [小爱同学启动～˶╹ꇴ╹˶！](https://www.bilibili.com/video/BV1YfVUz5EMj) + +### [C++ WebSocket ASR Server](https://github.com/mawwalker/stt-server) + +It provides a WebSocket server based on C++ for ASR using sherpa-onnx. + +### [Go WebSocket Server](https://github.com/bbeyondllove/asr_server) + +It provides a WebSocket server based on the Go programming language for sherpa-onnx. + +### [Making robot Paimon, Ep10 "The AI Part 1"](https://www.youtube.com/watch?v=KxPKkwxGWZs) + +It is a [YouTube video](https://www.youtube.com/watch?v=KxPKkwxGWZs), +showing how the author tried to use AI so he can have a conversation with Paimon. + +It uses sherpa-onnx for speech-to-text and text-to-speech. +|1| +|---| +|![](https://github.com/user-attachments/assets/f6eea2d5-1807-42cb-9160-be8da2971e1f)| + +### [TtsReader - Desktop application](https://github.com/ys-pro-duction/TtsReader) + +A desktop text-to-speech application built using Kotlin Multiplatform. + +### [MentraOS](https://github.com/Mentra-Community/MentraOS) + +> Smart glasses OS, with dozens of built-in apps. Users get AI assistant, notifications, +> translation, screen mirror, captions, and more. Devs get to write 1 app that runs on +> any pair of smart glasses. + +It uses sherpa-onnx for real-time speech recognition on iOS and Android devices. +See also + +It uses Swift for iOS and Java for Android. + +### [flet_sherpa_onnx](https://github.com/SamYuan1990/flet_sherpa_onnx) + +Flet ASR/STT component based on sherpa-onnx. +Example [a chat box agent](https://github.com/SamYuan1990/i18n-agent-action) + +### [achatbot-go](https://github.com/ai-bot-pro/achatbot-go) + +a multimodal chatbot based on go with sherpa-onnx's speech lib api. + +### [fcitx5-vinput](https://github.com/xifan2333/fcitx5-vinput) + +Local offline voice input plugin for [Fcitx5](https://github.com/fcitx/fcitx5) (Linux input method framework). +It uses C++ with offline ASR for speech recognition, supporting push-to-talk, +command mode, and optional LLM post-processing. + +Video demo in Chinese: [fcitx5-vinput](https://www.bilibili.com/video/BV1a6cUzVE6F) + +### [Wake Word](https://github.com/analyticsinmotion/wake-word) + +A VS Code extension for hands-free voice-activated coding. It uses sherpa-onnx for real-time +keyword spotting (KWS) to detect custom wake phrases and trigger VS Code commands by voice. +Audio capture is handled by [decibri](https://github.com/analyticsinmotion/decibri), a +cross-platform Node.js microphone streaming library with prebuilt native binaries. + +- [VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=analytics-in-motion.wake-word) +- [Open VSX](https://open-vsx.org/extension/analytics-in-motion/wake-word) +- [decibri integration guides for sherpa-onnx](https://decibri.dev/docs/node/integrations/sherpa-onnx-stt.html) + +### [SmartSub](https://github.com/buxuku/SmartSub) + +> SmartSub is a local-first cross-platform desktop application for the complete subtitle production pipeline: audio/video transcription, subtitle translation, proofreading, and subtitle burning/muxing. +> +> It natively integrates sherpa-onnx to power three offline ASR engines — FunASR, Qwen3-ASR, and FireRedASR — delivering high-accuracy Chinese and multilingual speech recognition entirely on-device, with no file uploads required. + +[silero-vad]: https://github.com/snakers4/silero-vad +[Raspberry Pi]: https://www.raspberrypi.com/ +[RV1126]: https://www.rock-chips.com/uploads/pdf/2022.8.26/191/RV1126%20Brief%20Datasheet.pdf +[LicheePi4A]: https://sipeed.com/licheepi4a +[VisionFive 2]: https://www.starfivetech.com/en/site/boards +[旭日X3派]: https://developer.horizon.ai/api/v1/fileData/documents_pi/index.html +[爱芯派]: https://wiki.sipeed.com/hardware/zh/maixIII/ax-pi/axpi.html +[hf-space-speaker-diarization]: https://huggingface.co/spaces/k2-fsa/speaker-diarization +[hf-space-speaker-diarization-cn]: https://hf.qhduan.com/spaces/k2-fsa/speaker-diarization +[hf-space-asr]: https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition +[hf-space-asr-cn]: https://hf.qhduan.com/spaces/k2-fsa/automatic-speech-recognition +[Whisper]: https://github.com/openai/whisper +[hf-space-asr-whisper]: https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition-with-whisper +[hf-space-asr-whisper-cn]: https://hf.qhduan.com/spaces/k2-fsa/automatic-speech-recognition-with-whisper +[hf-space-tts]: https://huggingface.co/spaces/k2-fsa/text-to-speech +[hf-space-tts-cn]: https://hf.qhduan.com/spaces/k2-fsa/text-to-speech +[hf-space-subtitle]: https://huggingface.co/spaces/k2-fsa/generate-subtitles-for-videos +[hf-space-subtitle-cn]: https://hf.qhduan.com/spaces/k2-fsa/generate-subtitles-for-videos +[hf-space-audio-tagging]: https://huggingface.co/spaces/k2-fsa/audio-tagging +[hf-space-audio-tagging-cn]: https://hf.qhduan.com/spaces/k2-fsa/audio-tagging +[hf-space-source-separation]: https://huggingface.co/spaces/k2-fsa/source-separation +[hf-space-source-separation-cn]: https://hf.qhduan.com/spaces/k2-fsa/source-separation +[hf-space-slid-whisper]: https://huggingface.co/spaces/k2-fsa/spoken-language-identification +[hf-space-slid-whisper-cn]: https://hf.qhduan.com/spaces/k2-fsa/spoken-language-identification +[wasm-hf-vad]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-sherpa-onnx +[wasm-ms-vad]: https://modelscope.cn/studios/csukuangfj/web-assembly-vad-sherpa-onnx +[wasm-hf-streaming-asr-zh-en-zipformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en +[wasm-ms-streaming-asr-zh-en-zipformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en +[wasm-hf-streaming-asr-zh-en-paraformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en-paraformer +[wasm-ms-streaming-asr-zh-en-paraformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en-paraformer +[Paraformer-large]: https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary +[wasm-hf-streaming-asr-zh-en-yue-paraformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-cantonese-en-paraformer +[wasm-ms-streaming-asr-zh-en-yue-paraformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-cantonese-en-paraformer +[wasm-hf-streaming-asr-en-zipformer]: https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-en +[wasm-ms-streaming-asr-en-zipformer]: https://modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-en +[SenseVoice]: https://github.com/FunAudioLLM/SenseVoice +[wasm-hf-vad-asr-zh-zipformer-ctc-07-03]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc +[wasm-ms-vad-asr-zh-zipformer-ctc-07-03]: https://modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc/summary +[wasm-hf-vad-asr-zh-en-ko-ja-yue-sense-voice]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-ja-ko-cantonese-sense-voice +[wasm-ms-vad-asr-zh-en-ko-ja-yue-sense-voice]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-zh-en-jp-ko-cantonese-sense-voice +[wasm-hf-vad-asr-en-whisper-tiny-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-whisper-tiny +[wasm-ms-vad-asr-en-whisper-tiny-en]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-en-whisper-tiny +[wasm-hf-vad-asr-en-moonshine-tiny-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny +[wasm-ms-vad-asr-en-moonshine-tiny-en]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny +[wasm-hf-vad-asr-en-zipformer-gigaspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-zipformer-gigaspeech +[wasm-ms-vad-asr-en-zipformer-gigaspeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-zipformer-gigaspeech +[wasm-hf-vad-asr-zh-zipformer-wenetspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-wenetspeech +[wasm-ms-vad-asr-zh-zipformer-wenetspeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-wenetspeech +[reazonspeech]: https://research.reazon.jp/_static/reazonspeech_nlp2023.pdf +[wasm-hf-vad-asr-ja-zipformer-reazonspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-ja-zipformer +[wasm-ms-vad-asr-ja-zipformer-reazonspeech]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-ja-zipformer +[gigaspeech2]: https://github.com/speechcolab/gigaspeech2 +[wasm-hf-vad-asr-th-zipformer-gigaspeech2]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-th-zipformer +[wasm-ms-vad-asr-th-zipformer-gigaspeech2]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-th-zipformer +[telespeech-asr]: https://github.com/tele-ai/telespeech-asr +[wasm-hf-vad-asr-zh-telespeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-telespeech +[wasm-ms-vad-asr-zh-telespeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-telespeech +[wasm-hf-vad-asr-zh-en-paraformer-large]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer +[wasm-ms-vad-asr-zh-en-paraformer-large]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer +[wasm-hf-vad-asr-zh-en-paraformer-small]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer-small +[wasm-ms-vad-asr-zh-en-paraformer-small]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-paraformer-small +[dolphin]: https://github.com/dataoceanai/dolphin +[wasm-ms-vad-asr-multi-lang-dolphin-base]: https://modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-multi-lang-dophin-ctc +[wasm-hf-vad-asr-multi-lang-dolphin-base]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-multi-lang-dophin-ctc + +[wasm-hf-tts-matcha-zh-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-en-tts-matcha +[wasm-hf-tts-matcha-zh]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-tts-matcha +[wasm-ms-tts-matcha-zh-en]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-en-tts-matcha +[wasm-ms-tts-matcha-zh]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-tts-matcha +[wasm-hf-tts-matcha-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-en-tts-matcha +[wasm-ms-tts-matcha-en]: https://modelscope.cn/studios/csukuangfj/web-assembly-en-tts-matcha +[wasm-hf-tts-piper-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-en +[wasm-ms-tts-piper-en]: https://modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-en +[wasm-hf-tts-piper-de]: https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-de +[wasm-ms-tts-piper-de]: https://modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-de +[wasm-hf-speaker-diarization]: https://huggingface.co/spaces/k2-fsa/web-assembly-speaker-diarization-sherpa-onnx +[wasm-ms-speaker-diarization]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-speaker-diarization-sherpa-onnx +[wasm-hf-voice-cloning-zipvoice]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-en-tts-zipvoice +[wasm-ms-voice-cloning-zipvoice]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-en-tts-zipvoice +[wasm-hf-voice-cloning-pocket]: https://huggingface.co/spaces/k2-fsa/web-assembly-en-tts-pocket +[wasm-ms-voice-cloning-pocket]: https://modelscope.cn/studios/csukuangfj/web-assembly-en-tts-pocket +[apk-speaker-diarization]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk.html +[apk-speaker-diarization-cn]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk-cn.html +[apk-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/android/apk.html +[apk-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-cn.html +[apk-simula-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/android/apk-simulate-streaming-asr.html +[apk-simula-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-simulate-streaming-asr-cn.html +[apk-tts]: https://k2-fsa.github.io/sherpa/onnx/tts/apk-engine.html +[apk-tts-cn]: https://k2-fsa.github.io/sherpa/onnx/tts/apk-engine-cn.html +[apk-vad]: https://k2-fsa.github.io/sherpa/onnx/vad/apk.html +[apk-vad-cn]: https://k2-fsa.github.io/sherpa/onnx/vad/apk-cn.html +[apk-vad-asr]: https://k2-fsa.github.io/sherpa/onnx/vad/apk-asr.html +[apk-vad-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/vad/apk-asr-cn.html +[apk-2pass]: https://k2-fsa.github.io/sherpa/onnx/android/apk-2pass.html +[apk-2pass-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-2pass-cn.html +[apk-at]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk.html +[apk-at-cn]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk-cn.html +[apk-at-wearos]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk-wearos.html +[apk-at-wearos-cn]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/apk-wearos-cn.html +[apk-sid]: https://k2-fsa.github.io/sherpa/onnx/speaker-identification/apk.html +[apk-sid-cn]: https://k2-fsa.github.io/sherpa/onnx/speaker-identification/apk-cn.html +[apk-slid]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/apk.html +[apk-slid-cn]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/apk-cn.html +[apk-kws]: https://k2-fsa.github.io/sherpa/onnx/kws/apk.html +[apk-kws-cn]: https://k2-fsa.github.io/sherpa/onnx/kws/apk-cn.html +[apk-flutter-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/flutter/pre-built-app.html#streaming-speech-recognition-stt-asr +[apk-flutter-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/pre-built-app.html#streaming-speech-recognition-stt-asr +[flutter-tts-android]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-android.html +[flutter-tts-android-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-android-cn.html +[flutter-tts-linux]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-linux.html +[flutter-tts-linux-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-linux-cn.html +[flutter-tts-macos-x64]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-x64.html +[flutter-tts-macos-arm64-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-arm64-cn.html +[flutter-tts-macos-arm64]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-arm64.html +[flutter-tts-macos-x64-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-macos-x64-cn.html +[flutter-tts-win-x64]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-win.html +[flutter-tts-win-x64-cn]: https://k2-fsa.github.io/sherpa/onnx/flutter/tts-win-cn.html +[lazarus-subtitle]: https://k2-fsa.github.io/sherpa/onnx/lazarus/download-generated-subtitles.html +[lazarus-subtitle-cn]: https://k2-fsa.github.io/sherpa/onnx/lazarus/download-generated-subtitles-cn.html +[asr-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models +[tts-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +[vad-models]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +[kws-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/kws-models +[at-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/audio-tagging-models +[sid-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +[slid-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +[punct-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/punctuation-models +[speaker-segmentation-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +[GigaSpeech]: https://github.com/SpeechColab/GigaSpeech +[WenetSpeech]: https://github.com/wenet-e2e/WenetSpeech +[sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +[sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16.tar.bz2 +[sherpa-onnx-streaming-zipformer-korean-2024-06-16]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-korean-2024-06-16.tar.bz2 +[sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2 +[sherpa-onnx-streaming-zipformer-en-20M-2023-02-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 +[sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01.tar.bz2 +[sherpa-onnx-zipformer-ru-2024-09-18]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ru-2024-09-18.tar.bz2 +[sherpa-onnx-zipformer-korean-2024-06-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-korean-2024-06-24.tar.bz2 +[sherpa-onnx-zipformer-thai-2024-06-20]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-thai-2024-06-20.tar.bz2 +[sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24.tar.bz2 +[sherpa-onnx-paraformer-zh-2024-03-09]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2024-03-09.tar.bz2 +[sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24.tar.bz2 +[sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04.tar.bz2 +[sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +[sherpa-onnx-streaming-zipformer-fr-2023-04-14]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-fr-2023-04-14.tar.bz2 +[Moonshine tiny]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +[NVIDIA Jetson Orin NX]: https://developer.download.nvidia.com/assets/embedded/secure/jetson/orin_nx/docs/Jetson_Orin_NX_DS-10712-001_v0.5.pdf?RCPGu9Q6OVAOv7a7vgtwc9-BLScXRIWq6cSLuditMALECJ_dOj27DgnqAPGVnT2VpiNpQan9SyFy-9zRykR58CokzbXwjSA7Gj819e91AXPrWkGZR3oS1VLxiDEpJa_Y0lr7UT-N4GnXtb8NlUkP4GkCkkF_FQivGPrAucCUywL481GH_WpP_p7ziHU1Wg==&t=eyJscyI6ImdzZW8iLCJsc2QiOiJodHRwczovL3d3dy5nb29nbGUuY29tLmhrLyJ9 +[NVIDIA Jetson Nano B01]: https://www.seeedstudio.com/blog/2020/01/16/new-revision-of-jetson-nano-dev-kit-now-supports-new-jetson-nano-module/ +[speech-enhancement-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models +[source-separation-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/source-separation-models +[RK3588]: https://www.rock-chips.com/uploads/pdf/2022.8.26/192/RK3588%20Brief%20Datasheet.pdf +[spleeter]: https://github.com/deezer/spleeter +[UVR]: https://github.com/Anjok07/ultimatevocalremovergui +[gtcrn]: https://github.com/Xiaobin-Rong/gtcrn +[tts-url]: https://k2-fsa.github.io/sherpa/onnx/tts/all-in-one.html +[ss-url]: https://k2-fsa.github.io/sherpa/onnx/source-separation/index.html +[sd-url]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/index.html +[slid-url]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/index.html +[at-url]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/index.html +[vad-url]: https://k2-fsa.github.io/sherpa/onnx/vad/index.html +[kws-url]: https://k2-fsa.github.io/sherpa/onnx/kws/index.html +[punct-url]: https://k2-fsa.github.io/sherpa/onnx/punctuation/index.html +[se-url]: https://k2-fsa.github.io/sherpa/onnx/speech-enhancement/index.html +[rknpu-doc]: https://k2-fsa.github.io/sherpa/onnx/rknn/index.html +[qnn-doc]: https://k2-fsa.github.io/sherpa/onnx/qnn/index.html +[ascend-doc]: https://k2-fsa.github.io/sherpa/onnx/ascend/index.html +[axera-npu]: https://axera-tech.com/Skill/166.html +[SpacemiT-K1]: https://cdn-resource.spacemit.com/file/chip/K1/K1_brief_zh.pdf +[SpacemiT-K3]: https://cdn-resource.spacemit.com/file/chip/K3/K3_brief_zh.pdf diff --git a/crates/sherpa-onnx-sys/build.rs b/crates/sherpa-onnx-sys/build.rs new file mode 100644 index 0000000000..274dfb293e --- /dev/null +++ b/crates/sherpa-onnx-sys/build.rs @@ -0,0 +1,475 @@ +use std::env; +use std::error::Error; +use std::ffi::OsStr; +use std::fs; +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; +use std::{collections::HashSet, ffi::OsString}; + +use bzip2::read::BzDecoder; +use tar::Archive; + +const RELEASE_BASE_URL: &str = "https://github.com/k2-fsa/sherpa-onnx/releases/download"; +const SHERPA_ONNX_STATIC_LIBS: &[&str] = &[ + "sherpa-onnx-c-api", + "sherpa-onnx-core", + "kaldi-decoder-core", + "sherpa-onnx-kaldifst-core", + "sherpa-onnx-fstfar", + "sherpa-onnx-fst", + "kaldi-native-fbank-core", + "kissfft-float", + "piper_phonemize", + "espeak-ng", + "ucd", + "onnxruntime", + "ssentencepiece_core", +]; + +type DynError = Box; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LinkMode { + Static, + Shared, +} + +fn main() { + if let Err(err) = try_main() { + panic!("{err}"); + } +} + +fn try_main() -> Result<(), DynError> { + println!("cargo:rerun-if-env-changed=SHERPA_ONNX_LIB_DIR"); + println!("cargo:rerun-if-env-changed=SHERPA_ONNX_ARCHIVE_DIR"); + println!("cargo:rerun-if-env-changed=DOCS_RS"); + + if env::var_os("DOCS_RS").is_some() { + // docs.rs sets DOCS_RS=1; skip downloading/linking native libraries + // so that `cargo doc` can succeed without the real C artifacts. + return Ok(()); + } + + let target_os = env::var("CARGO_CFG_TARGET_OS")?; + let target_arch = env::var("CARGO_CFG_TARGET_ARCH")?; + let link_mode = resolve_link_mode()?; + let lib_dir = resolve_lib_dir(link_mode, &target_os, &target_arch)?; + + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + + if link_mode == LinkMode::Shared && matches!(target_os.as_str(), "linux" | "macos") { + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display()); + emit_relative_rpath(&target_os); + copy_unix_runtime_libs(&lib_dir, &target_os)?; + } + + if link_mode == LinkMode::Shared && target_os == "windows" { + copy_windows_runtime_dlls(&lib_dir)?; + } + + match (link_mode, target_os.as_str()) { + (LinkMode::Static, "ios") => emit_ios_link_directives(), + (LinkMode::Shared, "android") => emit_android_link_directives(), + (LinkMode::Static, _) => emit_static_link_directives(&target_os), + (LinkMode::Shared, _) => emit_shared_link_directives(), + } + + Ok(()) +} + +fn resolve_link_mode() -> Result { + let static_enabled = env::var_os("CARGO_FEATURE_STATIC").is_some(); + let shared_enabled = env::var_os("CARGO_FEATURE_SHARED").is_some(); + + if static_enabled && shared_enabled { + return Err("Features `static` and `shared` cannot be enabled at the same time".into()); + } + + if shared_enabled { + Ok(LinkMode::Shared) + } else { + Ok(LinkMode::Static) + } +} + +fn resolve_lib_dir( + link_mode: LinkMode, + target_os: &str, + target_arch: &str, +) -> Result { + if let Some(path) = env::var_os("SHERPA_ONNX_LIB_DIR") { + let path = PathBuf::from(path); + if !path.is_dir() { + return Err(format!( + "SHERPA_ONNX_LIB_DIR does not exist or is not a directory: {}", + path.display() + ) + .into()); + } + return Ok(path); + } + + if matches!(target_os, "android" | "ios") { + return Err(format!( + "SHERPA_ONNX_LIB_DIR must point at the verified sherpa-onnx v{} \ + mobile libraries when building for os={target_os}, arch={target_arch}", + env!("CARGO_PKG_VERSION") + ) + .into()); + } + + download_prebuilt_libs(link_mode, target_os, target_arch) +} + +fn download_prebuilt_libs( + link_mode: LinkMode, + target_os: &str, + target_arch: &str, +) -> Result { + let archive_name = archive_name(link_mode, target_os, target_arch)?; + let archive_stem = archive_name.trim_end_matches(".tar.bz2"); + + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + let cache_root = target_dir_from_out_dir(&out_dir)?.join("sherpa-onnx-prebuilt"); + let extracted_dir = cache_root.join(archive_stem); + let lib_dir = extracted_dir.join("lib"); + + if lib_dir.is_dir() { + return Ok(lib_dir); + } + + fs::create_dir_all(&cache_root)?; + + let archive_path = cache_root.join(&archive_name); + if !archive_path.is_file() { + if let Some(local_archive_dir) = env::var_os("SHERPA_ONNX_ARCHIVE_DIR") { + let local_archive_path = PathBuf::from(local_archive_dir).join(&archive_name); + if !local_archive_path.is_file() { + return Err(format!( + "SHERPA_ONNX_ARCHIVE_DIR does not contain expected archive: {}", + local_archive_path.display() + ) + .into()); + } + + copy_file_atomically(&local_archive_path, &archive_path)?; + } else { + let version = env!("CARGO_PKG_VERSION"); + let url = format!("{RELEASE_BASE_URL}/v{version}/{archive_name}"); + eprintln!("Downloading sherpa-onnx libs from {url}"); + + let response = ureq::builder() + .try_proxy_from_env(true) + .build() + .get(&url) + .call() + .map_err(|e| format!("Failed to download sherpa-onnx archive from {url}: {e}"))?; + let mut reader = response.into_reader(); + write_reader_atomically(&mut reader, &archive_path)?; + } + } + + if extracted_dir.exists() { + fs::remove_dir_all(&extracted_dir)?; + } + + let unpack_result: Result<(), DynError> = (|| { + let tar_file = File::open(&archive_path)?; + let decoder = BzDecoder::new(tar_file); + let mut archive = Archive::new(decoder); + archive.unpack(&cache_root)?; + Ok(()) + })(); + if let Err(err) = unpack_result { + let _ = fs::remove_file(&archive_path); + let _ = fs::remove_dir_all(&extracted_dir); + return Err(format!( + "Failed to unpack cached archive {}: {err}", + archive_path.display() + ) + .into()); + } + + if !lib_dir.is_dir() { + return Err(format!( + "Downloaded archive did not contain a lib directory: {}", + lib_dir.display() + ) + .into()); + } + + eprintln!("Downloaded sherpa-onnx libs to {}", extracted_dir.display()); + + Ok(lib_dir) +} + +fn archive_name( + link_mode: LinkMode, + target_os: &str, + target_arch: &str, +) -> Result { + let version = env!("CARGO_PKG_VERSION"); + let name = match (link_mode, target_os, target_arch) { + (LinkMode::Static, "linux", "x86_64") => { + format!("sherpa-onnx-v{version}-linux-x64-static-lib.tar.bz2") + } + (LinkMode::Static, "linux", "aarch64") => { + format!("sherpa-onnx-v{version}-linux-aarch64-static-lib.tar.bz2") + } + (LinkMode::Static, "macos", "x86_64") => { + format!("sherpa-onnx-v{version}-osx-x64-static-lib.tar.bz2") + } + (LinkMode::Static, "macos", "aarch64") => { + format!("sherpa-onnx-v{version}-osx-arm64-static-lib.tar.bz2") + } + (LinkMode::Static, "windows", "x86_64") => { + format!("sherpa-onnx-v{version}-win-x64-static-MT-Release-lib.tar.bz2") + } + (LinkMode::Shared, "linux", "x86_64") => { + format!("sherpa-onnx-v{version}-linux-x64-shared-lib.tar.bz2") + } + (LinkMode::Shared, "linux", "aarch64") => { + format!("sherpa-onnx-v{version}-linux-aarch64-shared-cpu-lib.tar.bz2") + } + (LinkMode::Shared, "macos", "x86_64") => { + format!("sherpa-onnx-v{version}-osx-x64-shared-lib.tar.bz2") + } + (LinkMode::Shared, "macos", "aarch64") => { + format!("sherpa-onnx-v{version}-osx-arm64-shared-lib.tar.bz2") + } + (LinkMode::Shared, "windows", "x86_64") => { + format!("sherpa-onnx-v{version}-win-x64-shared-MT-Release-lib.tar.bz2") + } + _ => { + return Err(format!( + "Unsupported target for sherpa-onnx prebuilt libs: os={target_os}, arch={target_arch}" + ) + .into()) + } + }; + + Ok(name) +} + +fn emit_shared_link_directives() { + println!("cargo:rustc-link-lib=dylib=sherpa-onnx-c-api"); + println!("cargo:rustc-link-lib=dylib=onnxruntime"); +} + +fn emit_android_link_directives() { + println!("cargo:rustc-link-lib=dylib=sherpa-onnx-c-api"); + println!("cargo:rustc-link-lib=dylib=onnxruntime"); + println!("cargo:rustc-link-lib=dylib=log"); +} + +fn emit_ios_link_directives() { + println!("cargo:rustc-link-lib=static=sherpa-onnx"); + println!("cargo:rustc-link-lib=static=onnxruntime"); + println!("cargo:rustc-link-lib=dylib=c++"); + for framework in [ + "Accelerate", + "CoreML", + "Foundation", + "Metal", + "QuartzCore", + "Security", + ] { + println!("cargo:rustc-link-lib=framework={framework}"); + } +} + +fn emit_static_link_directives(target_os: &str) { + for lib in SHERPA_ONNX_STATIC_LIBS { + println!("cargo:rustc-link-lib=static={lib}"); + } + + match target_os { + "linux" => { + println!("cargo:rustc-link-lib=dylib=stdc++"); + println!("cargo:rustc-link-lib=dylib=m"); + println!("cargo:rustc-link-lib=dylib=pthread"); + println!("cargo:rustc-link-lib=dylib=dl"); + } + "macos" => { + println!("cargo:rustc-link-lib=dylib=c++"); + println!("cargo:rustc-link-lib=framework=Foundation"); + } + _ => {} + } +} + +fn target_dir_from_out_dir(out_dir: &Path) -> Result { + if let Ok(explicit_target_dir) = env::var("CARGO_TARGET_DIR") { + return Ok(PathBuf::from(explicit_target_dir)); + } + + if let Some(target_dir) = out_dir + .ancestors() + .find(|path| path.file_name() == Some(OsStr::new("target"))) + { + return Ok(target_dir.to_path_buf()); + } + + Ok(out_dir.to_path_buf()) +} + +fn emit_relative_rpath(target_os: &str) { + match target_os { + "linux" => println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN"), + "macos" => println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path"), + _ => {} + } +} + +fn profile_output_dirs() -> Result<[PathBuf; 2], DynError> { + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + let profile = env::var("PROFILE")?; + let profile_dir = out_dir + .ancestors() + .find(|path| path.file_name() == Some(OsStr::new(&profile))) + .ok_or_else(|| { + format!( + "Could not locate Cargo profile directory from {}", + out_dir.display() + ) + })? + .to_path_buf(); + + Ok([profile_dir.clone(), profile_dir.join("examples")]) +} + +fn copy_unix_runtime_libs(lib_dir: &Path, target_os: &str) -> Result<(), DynError> { + let runtime_libs: Vec = fs::read_dir(lib_dir)? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| { + path.file_name() + .and_then(OsStr::to_str) + .map(|name| match target_os { + "linux" => name.contains(".so"), + "macos" => name.ends_with(".dylib"), + _ => false, + }) + .unwrap_or(false) + }) + .collect(); + + if runtime_libs.is_empty() { + return Err(format!("No shared runtime libraries found in {}", lib_dir.display()).into()); + } + + let mut copy_plan = Vec::<(PathBuf, OsString)>::new(); + let mut planned_names = HashSet::::new(); + + for lib in runtime_libs { + if !lib.exists() { + continue; + } + + let lib_name = lib + .file_name() + .ok_or_else(|| format!("Invalid runtime library path: {}", lib.display()))? + .to_os_string(); + + let source = fs::canonicalize(&lib).unwrap_or(lib.clone()); + if planned_names.insert(lib_name.clone()) { + copy_plan.push((source.clone(), lib_name)); + } + + if let Some(source_name) = source.file_name() { + let source_name = source_name.to_os_string(); + if planned_names.insert(source_name.clone()) { + copy_plan.push((source.clone(), source_name)); + } + } + } + + if copy_plan.is_empty() { + return Err(format!( + "No usable shared runtime libraries found in {}", + lib_dir.display() + ) + .into()); + } + + for dest_dir in profile_output_dirs()? { + fs::create_dir_all(&dest_dir)?; + for (source, dest_name) in ©_plan { + let dest = dest_dir.join(dest_name); + fs::copy(source, &dest)?; + } + } + + Ok(()) +} + +fn temp_path_for(path: &Path) -> PathBuf { + let mut temp_name = path + .file_name() + .map(OsStr::to_os_string) + .unwrap_or_else(|| OsString::from("tmp")); + temp_name.push(".part"); + path.with_file_name(temp_name) +} + +fn copy_file_atomically(src: &Path, dst: &Path) -> Result<(), DynError> { + let temp_path = temp_path_for(dst); + if temp_path.exists() { + let _ = fs::remove_file(&temp_path); + } + fs::copy(src, &temp_path)?; + fs::rename(&temp_path, dst)?; + Ok(()) +} + +fn write_reader_atomically(reader: &mut dyn io::Read, dst: &Path) -> Result<(), DynError> { + let temp_path = temp_path_for(dst); + if temp_path.exists() { + let _ = fs::remove_file(&temp_path); + } + + { + let mut file = File::create(&temp_path)?; + io::copy(reader, &mut file)?; + file.sync_all()?; + } + + fs::rename(&temp_path, dst)?; + Ok(()) +} + +fn copy_windows_runtime_dlls(lib_dir: &Path) -> Result<(), DynError> { + let dlls: Vec = fs::read_dir(lib_dir)? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| path.extension() == Some(OsStr::new("dll"))) + .collect(); + + if dlls.is_empty() { + println!( + "cargo:warning=No runtime DLLs found in {}", + lib_dir.display() + ); + return Ok(()); + } + + let [profile_dir, examples_dir] = profile_output_dirs()?; + for dest_dir in [profile_dir.clone(), examples_dir] { + fs::create_dir_all(&dest_dir)?; + for dll in &dlls { + let dest = dest_dir.join( + dll.file_name() + .ok_or_else(|| format!("Invalid DLL path: {}", dll.display()))?, + ); + fs::copy(dll, &dest)?; + } + } + + println!( + "cargo:warning=Copied Windows runtime DLLs to {} and {}/examples", + profile_dir.display(), + profile_dir.display() + ); + + Ok(()) +} diff --git a/crates/sherpa-onnx-sys/src/audio_tagging.rs b/crates/sherpa-onnx-sys/src/audio_tagging.rs new file mode 100644 index 0000000000..6edf2d5d0a --- /dev/null +++ b/crates/sherpa-onnx-sys/src/audio_tagging.rs @@ -0,0 +1,60 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineZipformerAudioTaggingModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct AudioTaggingModelConfig { + pub zipformer: OfflineZipformerAudioTaggingModelConfig, + pub ced: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct AudioTaggingConfig { + pub model: AudioTaggingModelConfig, + pub labels: *const c_char, + pub top_k: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct AudioEvent { + pub name: *const c_char, + pub index: i32, + pub prob: c_float, +} + +#[repr(C)] +pub struct AudioTagging { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateAudioTagging(config: *const AudioTaggingConfig) -> *const AudioTagging; + + pub fn SherpaOnnxDestroyAudioTagging(tagger: *const AudioTagging); + + pub fn SherpaOnnxAudioTaggingCreateOfflineStream( + tagger: *const AudioTagging, + ) -> *const crate::offline_asr::OfflineStream; + + pub fn SherpaOnnxAudioTaggingCompute( + tagger: *const AudioTagging, + s: *const crate::offline_asr::OfflineStream, + top_k: i32, + ) -> *const *const AudioEvent; + + pub fn SherpaOnnxAudioTaggingFreeResults(p: *const *const AudioEvent); +} diff --git a/crates/sherpa-onnx-sys/src/kws.rs b/crates/sherpa-onnx-sys/src/kws.rs new file mode 100644 index 0000000000..25a2dd26c8 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/kws.rs @@ -0,0 +1,88 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KeywordResult { + pub keyword: *const c_char, + pub tokens: *const c_char, + pub tokens_arr: *const *const c_char, + pub count: i32, + pub timestamps: *mut c_float, + pub start_time: c_float, + pub json: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KeywordSpotterConfig { + pub feat_config: super::online_asr::FeatureConfig, + pub model_config: super::online_asr::OnlineModelConfig, + pub max_active_paths: i32, + pub num_trailing_blanks: i32, + pub keywords_score: c_float, + pub keywords_threshold: c_float, + pub keywords_file: *const c_char, + pub keywords_buf: *const c_char, + pub keywords_buf_size: i32, +} + +#[repr(C)] +pub struct KeywordSpotter { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateKeywordSpotter( + config: *const KeywordSpotterConfig, + ) -> *const KeywordSpotter; + + pub fn SherpaOnnxDestroyKeywordSpotter(spotter: *const KeywordSpotter); + + pub fn SherpaOnnxCreateKeywordStream( + spotter: *const KeywordSpotter, + ) -> *const super::online_asr::OnlineStream; + + pub fn SherpaOnnxCreateKeywordStreamWithKeywords( + spotter: *const KeywordSpotter, + keywords: *const c_char, + ) -> *const super::online_asr::OnlineStream; + + pub fn SherpaOnnxIsKeywordStreamReady( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ) -> i32; + + pub fn SherpaOnnxDecodeKeywordStream( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ); + + pub fn SherpaOnnxResetKeywordStream( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ); + + pub fn SherpaOnnxDecodeMultipleKeywordStreams( + spotter: *const KeywordSpotter, + streams: *const *const super::online_asr::OnlineStream, + n: i32, + ); + + pub fn SherpaOnnxGetKeywordResult( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ) -> *const KeywordResult; + + pub fn SherpaOnnxDestroyKeywordResult(r: *const KeywordResult); + + pub fn SherpaOnnxGetKeywordResultAsJson( + spotter: *const KeywordSpotter, + stream: *const super::online_asr::OnlineStream, + ) -> *const c_char; + + pub fn SherpaOnnxFreeKeywordResultJson(s: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/lib.rs b/crates/sherpa-onnx-sys/src/lib.rs new file mode 100644 index 0000000000..b4161cc616 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/lib.rs @@ -0,0 +1,42 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +extern "C" { + pub fn SherpaOnnxGetVersionStr() -> *const c_char; + pub fn SherpaOnnxGetGitSha1() -> *const c_char; + pub fn SherpaOnnxGetGitDate() -> *const c_char; + pub fn SherpaOnnxFileExists(filename: *const c_char) -> i32; +} + +pub mod audio_tagging; +pub mod kws; +pub mod offline_asr; +pub mod offline_punctuation; +pub mod offline_speaker_diarization; +pub mod online_asr; +pub mod online_punctuation; +pub mod resampler; +pub mod speaker_embedding; +pub mod speech_denoiser; +pub mod spoken_language_identification; +pub mod tts; +pub mod vad; +pub mod wave; + +pub use audio_tagging::*; +pub use kws::*; +pub use offline_asr::*; +pub use offline_punctuation::*; +pub use offline_speaker_diarization::*; +pub use online_asr::*; +pub use online_punctuation::*; +pub use resampler::*; +pub use speaker_embedding::*; +pub use speech_denoiser::*; +pub use spoken_language_identification::*; +pub use tts::*; +pub use vad::*; +pub use wave::*; diff --git a/crates/sherpa-onnx-sys/src/offline_asr.rs b/crates/sherpa-onnx-sys/src/offline_asr.rs new file mode 100644 index 0000000000..9d0d28c748 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/offline_asr.rs @@ -0,0 +1,281 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTransducerModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub joiner: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineParaformerModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineNemoEncDecCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineWhisperModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub language: *const c_char, + pub task: *const c_char, + pub tail_paddings: i32, + pub enable_token_timestamps: i32, + pub enable_segment_timestamps: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineCanaryModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub src_lang: *const c_char, + pub tgt_lang: *const c_char, + pub use_pnc: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineFireRedAsrModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineMoonshineModelConfig { + pub preprocessor: *const c_char, + pub encoder: *const c_char, + pub uncached_decoder: *const c_char, + pub cached_decoder: *const c_char, + pub merged_decoder: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTdnnModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineLMConfig { + pub model: *const c_char, + pub scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSenseVoiceModelConfig { + pub model: *const c_char, + pub language: *const c_char, + pub use_itn: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineDolphinModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineZipformerCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineWenetCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineOmnilingualAsrCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineFunASRNanoModelConfig { + pub encoder_adaptor: *const c_char, + pub llm: *const c_char, + pub embedding: *const c_char, + pub tokenizer: *const c_char, + pub system_prompt: *const c_char, + pub user_prompt: *const c_char, + pub max_new_tokens: i32, + pub temperature: c_float, + pub top_p: c_float, + pub seed: i32, + pub language: *const c_char, + pub itn: i32, + pub hotwords: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineMedAsrCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineFireRedAsrCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineQwen3ASRModelConfig { + pub conv_frontend: *const c_char, + pub encoder: *const c_char, + pub decoder: *const c_char, + pub tokenizer: *const c_char, + pub max_total_len: i32, + pub max_new_tokens: i32, + pub temperature: c_float, + pub top_p: c_float, + pub seed: i32, + pub hotwords: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineCohereTranscribeModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub language: *const c_char, + pub use_punct: i32, + pub use_itn: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineModelConfig { + pub transducer: OfflineTransducerModelConfig, + pub paraformer: OfflineParaformerModelConfig, + pub nemo_ctc: OfflineNemoEncDecCtcModelConfig, + pub whisper: OfflineWhisperModelConfig, + pub tdnn: OfflineTdnnModelConfig, + + pub tokens: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, + pub model_type: *const c_char, + pub modeling_unit: *const c_char, + pub bpe_vocab: *const c_char, + pub telespeech_ctc: *const c_char, + + pub sense_voice: OfflineSenseVoiceModelConfig, + pub moonshine: OfflineMoonshineModelConfig, + pub fire_red_asr: OfflineFireRedAsrModelConfig, + pub dolphin: OfflineDolphinModelConfig, + pub zipformer_ctc: OfflineZipformerCtcModelConfig, + pub canary: OfflineCanaryModelConfig, + pub wenet_ctc: OfflineWenetCtcModelConfig, + pub omnilingual: OfflineOmnilingualAsrCtcModelConfig, + pub medasr: OfflineMedAsrCtcModelConfig, + pub funasr_nano: OfflineFunASRNanoModelConfig, + pub fire_red_asr_ctc: OfflineFireRedAsrCtcModelConfig, + pub qwen3_asr: OfflineQwen3ASRModelConfig, + pub cohere_transcribe: OfflineCohereTranscribeModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineRecognizerConfig { + pub feat_config: super::online_asr::FeatureConfig, + pub model_config: OfflineModelConfig, + pub lm_config: OfflineLMConfig, + + pub decoding_method: *const c_char, + pub max_active_paths: i32, + pub hotwords_file: *const c_char, + pub hotwords_score: c_float, + pub rule_fsts: *const c_char, + pub rule_fars: *const c_char, + pub blank_penalty: c_float, + pub hr: super::online_asr::HomophoneReplacerConfig, +} + +#[repr(C)] +pub struct OfflineRecognizer { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OfflineStream { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineRecognizer( + config: *const OfflineRecognizerConfig, + ) -> *const OfflineRecognizer; + + pub fn SherpaOnnxDestroyOfflineRecognizer(recognizer: *const OfflineRecognizer); + + pub fn SherpaOnnxCreateOfflineStream( + recognizer: *const OfflineRecognizer, + ) -> *const OfflineStream; + + pub fn SherpaOnnxCreateOfflineStreamWithHotwords( + recognizer: *const OfflineRecognizer, + hotwords: *const c_char, + ) -> *const OfflineStream; + + pub fn SherpaOnnxDestroyOfflineStream(stream: *const OfflineStream); + + pub fn SherpaOnnxAcceptWaveformOffline( + stream: *const OfflineStream, + sample_rate: i32, + samples: *const f32, + n: i32, + ); + + pub fn SherpaOnnxOfflineStreamSetOption( + stream: *const OfflineStream, + key: *const c_char, + value: *const c_char, + ); + + pub fn SherpaOnnxOfflineStreamGetOption( + stream: *const OfflineStream, + key: *const c_char, + ) -> *const c_char; + + pub fn SherpaOnnxOfflineStreamHasOption( + stream: *const OfflineStream, + key: *const c_char, + ) -> i32; + + pub fn SherpaOnnxDecodeOfflineStream( + recognizer: *const OfflineRecognizer, + stream: *const OfflineStream, + ); + + pub fn SherpaOnnxDecodeMultipleOfflineStreams( + recognizer: *const OfflineRecognizer, + streams: *const *const OfflineStream, + n: i32, + ); + + pub fn SherpaOnnxGetOfflineStreamResultAsJson(stream: *const OfflineStream) -> *const c_char; + + pub fn SherpaOnnxDestroyOfflineStreamResultJson(s: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/offline_punctuation.rs b/crates/sherpa-onnx-sys/src/offline_punctuation.rs new file mode 100644 index 0000000000..06547846cf --- /dev/null +++ b/crates/sherpa-onnx-sys/src/offline_punctuation.rs @@ -0,0 +1,40 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflinePunctuationModelConfig { + pub ct_transformer: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflinePunctuationConfig { + pub model: OfflinePunctuationModelConfig, +} + +#[repr(C)] +pub struct OfflinePunctuation { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflinePunctuation( + config: *const OfflinePunctuationConfig, + ) -> *const OfflinePunctuation; + + pub fn SherpaOnnxDestroyOfflinePunctuation(punct: *const OfflinePunctuation); + + pub fn SherpaOfflinePunctuationAddPunct( + punct: *const OfflinePunctuation, + text: *const c_char, + ) -> *const c_char; + + pub fn SherpaOfflinePunctuationFreeText(text: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs b/crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs new file mode 100644 index 0000000000..00232e0a98 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/offline_speaker_diarization.rs @@ -0,0 +1,98 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerSegmentationPyannoteModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerSegmentationModelConfig { + pub pyannote: OfflineSpeakerSegmentationPyannoteModelConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct FastClusteringConfig { + pub num_clusters: i32, + pub threshold: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerDiarizationConfig { + pub segmentation: OfflineSpeakerSegmentationModelConfig, + pub embedding: crate::speaker_embedding::SpeakerEmbeddingExtractorConfig, + pub clustering: FastClusteringConfig, + pub min_duration_on: c_float, + pub min_duration_off: c_float, +} + +#[repr(C)] +pub struct OfflineSpeakerDiarization { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OfflineSpeakerDiarizationResult { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeakerDiarizationSegment { + pub start: c_float, + pub end: c_float, + pub speaker: i32, +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineSpeakerDiarization( + config: *const OfflineSpeakerDiarizationConfig, + ) -> *const OfflineSpeakerDiarization; + + pub fn SherpaOnnxDestroyOfflineSpeakerDiarization(sd: *const OfflineSpeakerDiarization); + + pub fn SherpaOnnxOfflineSpeakerDiarizationGetSampleRate( + sd: *const OfflineSpeakerDiarization, + ) -> i32; + + pub fn SherpaOnnxOfflineSpeakerDiarizationSetConfig( + sd: *const OfflineSpeakerDiarization, + config: *const OfflineSpeakerDiarizationConfig, + ); + + pub fn SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers( + r: *const OfflineSpeakerDiarizationResult, + ) -> i32; + + pub fn SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments( + r: *const OfflineSpeakerDiarizationResult, + ) -> i32; + + pub fn SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime( + r: *const OfflineSpeakerDiarizationResult, + ) -> *const OfflineSpeakerDiarizationSegment; + + pub fn SherpaOnnxOfflineSpeakerDiarizationDestroySegment( + s: *const OfflineSpeakerDiarizationSegment, + ); + + pub fn SherpaOnnxOfflineSpeakerDiarizationProcess( + sd: *const OfflineSpeakerDiarization, + samples: *const c_float, + n: i32, + ) -> *const OfflineSpeakerDiarizationResult; + + pub fn SherpaOnnxOfflineSpeakerDiarizationDestroyResult( + r: *const OfflineSpeakerDiarizationResult, + ); +} diff --git a/crates/sherpa-onnx-sys/src/online_asr.rs b/crates/sherpa-onnx-sys/src/online_asr.rs new file mode 100644 index 0000000000..522f934d79 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/online_asr.rs @@ -0,0 +1,202 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineTransducerModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub joiner: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineParaformerModelConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineZipformer2CtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineNemoCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineToneCtcModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineModelConfig { + pub transducer: OnlineTransducerModelConfig, + pub paraformer: OnlineParaformerModelConfig, + pub zipformer2_ctc: OnlineZipformer2CtcModelConfig, + + pub tokens: *const c_char, + pub num_threads: i32, + pub provider: *const c_char, + pub debug: i32, + + pub model_type: *const c_char, + + // cjkchar | bpe | cjkchar+bpe + pub modeling_unit: *const c_char, + + pub bpe_vocab: *const c_char, + + pub tokens_buf: *const u8, + pub tokens_buf_size: i32, + + pub nemo_ctc: OnlineNemoCtcModelConfig, + pub t_one_ctc: OnlineToneCtcModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct FeatureConfig { + pub sample_rate: i32, + pub feature_dim: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineCtcFstDecoderConfig { + pub graph: *const c_char, + pub max_active: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct HomophoneReplacerConfig { + pub dict_dir: *const c_char, + pub lexicon: *const c_char, + pub rule_fsts: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineRecognizerConfig { + pub feat_config: FeatureConfig, + pub model_config: OnlineModelConfig, + + // greedy_search | modified_beam_search + pub decoding_method: *const c_char, + + pub max_active_paths: i32, + + pub enable_endpoint: i32, + + pub rule1_min_trailing_silence: c_float, + pub rule2_min_trailing_silence: c_float, + pub rule3_min_utterance_length: c_float, + + pub hotwords_file: *const c_char, + pub hotwords_score: c_float, + + pub ctc_fst_decoder_config: OnlineCtcFstDecoderConfig, + + pub rule_fsts: *const c_char, + pub rule_fars: *const c_char, + + pub blank_penalty: c_float, + + pub hotwords_buf: *const u8, + pub hotwords_buf_size: i32, + + pub hr: HomophoneReplacerConfig, +} + +#[repr(C)] +pub struct OnlineRecognizer { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OnlineStream { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOnlineRecognizer( + config: *const OnlineRecognizerConfig, + ) -> *const OnlineRecognizer; + + pub fn SherpaOnnxDestroyOnlineRecognizer(recognizer: *const OnlineRecognizer); + + pub fn SherpaOnnxCreateOnlineStream(recognizer: *const OnlineRecognizer) + -> *const OnlineStream; + + pub fn SherpaOnnxCreateOnlineStreamWithHotwords( + recognizer: *const OnlineRecognizer, + hotwords: *const c_char, + ) -> *const OnlineStream; + + pub fn SherpaOnnxDestroyOnlineStream(stream: *const OnlineStream); + + pub fn SherpaOnnxOnlineStreamAcceptWaveform( + stream: *const OnlineStream, + sample_rate: i32, + samples: *const f32, + n: i32, + ); + + pub fn SherpaOnnxIsOnlineStreamReady( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ) -> i32; + + pub fn SherpaOnnxDecodeOnlineStream( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ); + + pub fn SherpaOnnxDecodeMultipleOnlineStreams( + recognizer: *const OnlineRecognizer, + streams: *const *const OnlineStream, + n: i32, + ); + + pub fn SherpaOnnxGetOnlineStreamResultAsJson( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ) -> *const c_char; + + pub fn SherpaOnnxDestroyOnlineStreamResultJson(s: *const c_char); + + pub fn SherpaOnnxOnlineStreamReset( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ); + + pub fn SherpaOnnxOnlineStreamInputFinished(stream: *const OnlineStream); + + pub fn SherpaOnnxOnlineStreamSetOption( + stream: *const OnlineStream, + key: *const c_char, + value: *const c_char, + ); + + pub fn SherpaOnnxOnlineStreamGetOption( + stream: *const OnlineStream, + key: *const c_char, + ) -> *const c_char; + + pub fn SherpaOnnxOnlineStreamHasOption(stream: *const OnlineStream, key: *const c_char) -> i32; + + pub fn SherpaOnnxOnlineStreamIsEndpoint( + recognizer: *const OnlineRecognizer, + stream: *const OnlineStream, + ) -> i32; +} diff --git a/crates/sherpa-onnx-sys/src/online_punctuation.rs b/crates/sherpa-onnx-sys/src/online_punctuation.rs new file mode 100644 index 0000000000..1ff1fce1b0 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/online_punctuation.rs @@ -0,0 +1,41 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlinePunctuationModelConfig { + pub cnn_bilstm: *const c_char, + pub bpe_vocab: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlinePunctuationConfig { + pub model: OnlinePunctuationModelConfig, +} + +#[repr(C)] +pub struct OnlinePunctuation { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOnlinePunctuation( + config: *const OnlinePunctuationConfig, + ) -> *const OnlinePunctuation; + + pub fn SherpaOnnxDestroyOnlinePunctuation(punctuation: *const OnlinePunctuation); + + pub fn SherpaOnnxOnlinePunctuationAddPunct( + punctuation: *const OnlinePunctuation, + text: *const c_char, + ) -> *const c_char; + + pub fn SherpaOnnxOnlinePunctuationFreeText(text: *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/resampler.rs b/crates/sherpa-onnx-sys/src/resampler.rs new file mode 100644 index 0000000000..fd49835662 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/resampler.rs @@ -0,0 +1,42 @@ +use std::os::raw::c_float; + +#[repr(C)] +pub struct SherpaOnnxLinearResampler { + _private: [u8; 0], +} + +#[repr(C)] +pub struct SherpaOnnxResampleOut { + pub samples: *const f32, + pub n: i32, +} + +extern "C" { + pub fn SherpaOnnxCreateLinearResampler( + samp_rate_in_hz: i32, + samp_rate_out_hz: i32, + filter_cutoff_hz: c_float, + num_zeros: i32, + ) -> *const SherpaOnnxLinearResampler; + + pub fn SherpaOnnxDestroyLinearResampler(p: *const SherpaOnnxLinearResampler); + + pub fn SherpaOnnxLinearResamplerReset(p: *const SherpaOnnxLinearResampler); + + pub fn SherpaOnnxLinearResamplerResample( + p: *const SherpaOnnxLinearResampler, + input: *const f32, + input_dim: i32, + flush: i32, + ) -> *const SherpaOnnxResampleOut; + + pub fn SherpaOnnxLinearResamplerResampleFree(p: *const SherpaOnnxResampleOut); + + pub fn SherpaOnnxLinearResamplerResampleGetInputSampleRate( + p: *const SherpaOnnxLinearResampler, + ) -> i32; + + pub fn SherpaOnnxLinearResamplerResampleGetOutputSampleRate( + p: *const SherpaOnnxLinearResampler, + ) -> i32; +} diff --git a/crates/sherpa-onnx-sys/src/speaker_embedding.rs b/crates/sherpa-onnx-sys/src/speaker_embedding.rs new file mode 100644 index 0000000000..ca745e3106 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/speaker_embedding.rs @@ -0,0 +1,131 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpeakerEmbeddingExtractorConfig { + pub model: *const c_char, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +pub struct SpeakerEmbeddingExtractor { + _private: [u8; 0], +} + +#[repr(C)] +pub struct SpeakerEmbeddingManager { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpeakerEmbeddingManagerSpeakerMatch { + pub score: c_float, + pub name: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpeakerEmbeddingManagerBestMatchesResult { + pub matches: *const SpeakerEmbeddingManagerSpeakerMatch, + pub count: i32, +} + +extern "C" { + pub fn SherpaOnnxCreateSpeakerEmbeddingExtractor( + config: *const SpeakerEmbeddingExtractorConfig, + ) -> *const SpeakerEmbeddingExtractor; + + pub fn SherpaOnnxDestroySpeakerEmbeddingExtractor(p: *const SpeakerEmbeddingExtractor); + + pub fn SherpaOnnxSpeakerEmbeddingExtractorDim(p: *const SpeakerEmbeddingExtractor) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorCreateStream( + p: *const SpeakerEmbeddingExtractor, + ) -> *const crate::online_asr::OnlineStream; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorIsReady( + p: *const SpeakerEmbeddingExtractor, + s: *const crate::online_asr::OnlineStream, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorComputeEmbedding( + p: *const SpeakerEmbeddingExtractor, + s: *const crate::online_asr::OnlineStream, + ) -> *const c_float; + + pub fn SherpaOnnxSpeakerEmbeddingExtractorDestroyEmbedding(v: *const c_float); + + pub fn SherpaOnnxCreateSpeakerEmbeddingManager(dim: i32) -> *const SpeakerEmbeddingManager; + + pub fn SherpaOnnxDestroySpeakerEmbeddingManager(p: *const SpeakerEmbeddingManager); + + pub fn SherpaOnnxSpeakerEmbeddingManagerAdd( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const c_float, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerAddList( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const *const c_float, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerAddListFlattened( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const c_float, + n: i32, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerRemove( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerSearch( + p: *const SpeakerEmbeddingManager, + v: *const c_float, + threshold: c_float, + ) -> *const c_char; + + pub fn SherpaOnnxSpeakerEmbeddingManagerFreeSearch(name: *const c_char); + + pub fn SherpaOnnxSpeakerEmbeddingManagerGetBestMatches( + p: *const SpeakerEmbeddingManager, + v: *const c_float, + threshold: c_float, + n: i32, + ) -> *const SpeakerEmbeddingManagerBestMatchesResult; + + pub fn SherpaOnnxSpeakerEmbeddingManagerFreeBestMatches( + r: *const SpeakerEmbeddingManagerBestMatchesResult, + ); + + pub fn SherpaOnnxSpeakerEmbeddingManagerVerify( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + v: *const c_float, + threshold: c_float, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerContains( + p: *const SpeakerEmbeddingManager, + name: *const c_char, + ) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerNumSpeakers(p: *const SpeakerEmbeddingManager) -> i32; + + pub fn SherpaOnnxSpeakerEmbeddingManagerGetAllSpeakers( + p: *const SpeakerEmbeddingManager, + ) -> *const *const c_char; + + pub fn SherpaOnnxSpeakerEmbeddingManagerFreeAllSpeakers(names: *const *const c_char); +} diff --git a/crates/sherpa-onnx-sys/src/speech_denoiser.rs b/crates/sherpa-onnx-sys/src/speech_denoiser.rs new file mode 100644 index 0000000000..dcf7f1e0e4 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/speech_denoiser.rs @@ -0,0 +1,88 @@ +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserGtcrnModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserDpdfNetModelConfig { + pub model: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserModelConfig { + pub gtcrn: OfflineSpeechDenoiserGtcrnModelConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, + pub dpdfnet: OfflineSpeechDenoiserDpdfNetModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineSpeechDenoiserConfig { + pub model: OfflineSpeechDenoiserModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OnlineSpeechDenoiserConfig { + pub model: OfflineSpeechDenoiserModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct DenoisedAudio { + pub samples: *const f32, + pub n: i32, + pub sample_rate: i32, +} + +#[repr(C)] +pub struct OfflineSpeechDenoiser { + _private: [u8; 0], +} + +#[repr(C)] +pub struct OnlineSpeechDenoiser { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineSpeechDenoiser( + config: *const OfflineSpeechDenoiserConfig, + ) -> *const OfflineSpeechDenoiser; + pub fn SherpaOnnxDestroyOfflineSpeechDenoiser(p: *const OfflineSpeechDenoiser); + pub fn SherpaOnnxOfflineSpeechDenoiserGetSampleRate(p: *const OfflineSpeechDenoiser) -> i32; + pub fn SherpaOnnxOfflineSpeechDenoiserRun( + p: *const OfflineSpeechDenoiser, + samples: *const f32, + n: i32, + sample_rate: i32, + ) -> *const DenoisedAudio; + + pub fn SherpaOnnxCreateOnlineSpeechDenoiser( + config: *const OnlineSpeechDenoiserConfig, + ) -> *const OnlineSpeechDenoiser; + pub fn SherpaOnnxDestroyOnlineSpeechDenoiser(p: *const OnlineSpeechDenoiser); + pub fn SherpaOnnxOnlineSpeechDenoiserGetSampleRate(p: *const OnlineSpeechDenoiser) -> i32; + pub fn SherpaOnnxOnlineSpeechDenoiserGetFrameShiftInSamples( + p: *const OnlineSpeechDenoiser, + ) -> i32; + pub fn SherpaOnnxOnlineSpeechDenoiserRun( + p: *const OnlineSpeechDenoiser, + samples: *const f32, + n: i32, + sample_rate: i32, + ) -> *const DenoisedAudio; + pub fn SherpaOnnxOnlineSpeechDenoiserFlush( + p: *const OnlineSpeechDenoiser, + ) -> *const DenoisedAudio; + pub fn SherpaOnnxOnlineSpeechDenoiserReset(p: *const OnlineSpeechDenoiser); + + pub fn SherpaOnnxDestroyDenoisedAudio(audio: *const DenoisedAudio); +} diff --git a/crates/sherpa-onnx-sys/src/spoken_language_identification.rs b/crates/sherpa-onnx-sys/src/spoken_language_identification.rs new file mode 100644 index 0000000000..f9d8f46753 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/spoken_language_identification.rs @@ -0,0 +1,54 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpokenLanguageIdentificationWhisperConfig { + pub encoder: *const c_char, + pub decoder: *const c_char, + pub tail_paddings: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpokenLanguageIdentificationConfig { + pub whisper: SpokenLanguageIdentificationWhisperConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, +} + +#[repr(C)] +pub struct SpokenLanguageIdentification { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SpokenLanguageIdentificationResult { + pub lang: *const c_char, +} + +extern "C" { + pub fn SherpaOnnxCreateSpokenLanguageIdentification( + config: *const SpokenLanguageIdentificationConfig, + ) -> *const SpokenLanguageIdentification; + + pub fn SherpaOnnxDestroySpokenLanguageIdentification(slid: *const SpokenLanguageIdentification); + + pub fn SherpaOnnxSpokenLanguageIdentificationCreateOfflineStream( + slid: *const SpokenLanguageIdentification, + ) -> *const super::offline_asr::OfflineStream; + + pub fn SherpaOnnxSpokenLanguageIdentificationCompute( + slid: *const SpokenLanguageIdentification, + stream: *const super::offline_asr::OfflineStream, + ) -> *const SpokenLanguageIdentificationResult; + + pub fn SherpaOnnxDestroySpokenLanguageIdentificationResult( + r: *const SpokenLanguageIdentificationResult, + ); +} diff --git a/crates/sherpa-onnx-sys/src/tts.rs b/crates/sherpa-onnx-sys/src/tts.rs new file mode 100644 index 0000000000..6be479c56a --- /dev/null +++ b/crates/sherpa-onnx-sys/src/tts.rs @@ -0,0 +1,172 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_float, c_void}; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsVitsModelConfig { + pub model: *const c_char, + pub lexicon: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub noise_scale: c_float, + pub noise_scale_w: c_float, + pub length_scale: c_float, + pub dict_dir: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsMatchaModelConfig { + pub acoustic_model: *const c_char, + pub vocoder: *const c_char, + pub lexicon: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub noise_scale: c_float, + pub length_scale: c_float, + pub dict_dir: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsKokoroModelConfig { + pub model: *const c_char, + pub voices: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub length_scale: c_float, + pub dict_dir: *const c_char, + pub lexicon: *const c_char, + pub lang: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsKittenModelConfig { + pub model: *const c_char, + pub voices: *const c_char, + pub tokens: *const c_char, + pub data_dir: *const c_char, + pub length_scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsZipvoiceModelConfig { + pub tokens: *const c_char, + pub encoder: *const c_char, + pub decoder: *const c_char, + pub vocoder: *const c_char, + pub data_dir: *const c_char, + pub lexicon: *const c_char, + pub feat_scale: c_float, + pub t_shift: c_float, + pub target_rms: c_float, + pub guidance_scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsPocketModelConfig { + pub lm_flow: *const c_char, + pub lm_main: *const c_char, + pub encoder: *const c_char, + pub decoder: *const c_char, + pub text_conditioner: *const c_char, + pub vocab_json: *const c_char, + pub token_scores_json: *const c_char, + pub voice_embedding_cache_capacity: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsSupertonicModelConfig { + pub duration_predictor: *const c_char, + pub text_encoder: *const c_char, + pub vector_estimator: *const c_char, + pub vocoder: *const c_char, + pub tts_json: *const c_char, + pub unicode_indexer: *const c_char, + pub voice_style: *const c_char, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsModelConfig { + pub vits: OfflineTtsVitsModelConfig, + pub num_threads: i32, + pub debug: i32, + pub provider: *const c_char, + pub matcha: OfflineTtsMatchaModelConfig, + pub kokoro: OfflineTtsKokoroModelConfig, + pub kitten: OfflineTtsKittenModelConfig, + pub zipvoice: OfflineTtsZipvoiceModelConfig, + pub pocket: OfflineTtsPocketModelConfig, + pub supertonic: OfflineTtsSupertonicModelConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineTtsConfig { + pub model: OfflineTtsModelConfig, + pub rule_fsts: *const c_char, + pub max_num_sentences: i32, + pub rule_fars: *const c_char, + pub silence_scale: c_float, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SherpaOnnxGeneratedAudio { + pub samples: *const f32, + pub n: i32, + pub sample_rate: i32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SherpaOnnxGenerationConfig { + pub silence_scale: c_float, + pub speed: c_float, + pub sid: i32, + pub reference_audio: *const f32, + pub reference_audio_len: i32, + pub reference_sample_rate: i32, + pub reference_text: *const c_char, + pub num_steps: i32, + pub extra: *const c_char, +} + +pub type SherpaOnnxGeneratedAudioProgressCallbackWithArg = Option< + unsafe extern "C" fn(samples: *const f32, n: i32, progress: c_float, arg: *mut c_void) -> i32, +>; + +#[repr(C)] +pub struct SherpaOnnxOfflineTts { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateOfflineTts( + config: *const OfflineTtsConfig, + ) -> *const SherpaOnnxOfflineTts; + + pub fn SherpaOnnxDestroyOfflineTts(tts: *const SherpaOnnxOfflineTts); + + pub fn SherpaOnnxOfflineTtsSampleRate(tts: *const SherpaOnnxOfflineTts) -> i32; + + pub fn SherpaOnnxOfflineTtsNumSpeakers(tts: *const SherpaOnnxOfflineTts) -> i32; + + pub fn SherpaOnnxOfflineTtsGenerateWithConfig( + tts: *const SherpaOnnxOfflineTts, + text: *const c_char, + config: *const SherpaOnnxGenerationConfig, + callback: SherpaOnnxGeneratedAudioProgressCallbackWithArg, + arg: *mut c_void, + ) -> *const SherpaOnnxGeneratedAudio; + + pub fn SherpaOnnxDestroyOfflineTtsGeneratedAudio(p: *const SherpaOnnxGeneratedAudio); +} diff --git a/crates/sherpa-onnx-sys/src/vad.rs b/crates/sherpa-onnx-sys/src/vad.rs new file mode 100644 index 0000000000..b16b21deb9 --- /dev/null +++ b/crates/sherpa-onnx-sys/src/vad.rs @@ -0,0 +1,85 @@ +use std::os::raw::{c_char, c_float}; + +#[repr(C)] +pub struct SileroVadModelConfig { + pub model: *const c_char, + pub threshold: c_float, + pub min_silence_duration: c_float, + pub min_speech_duration: c_float, + pub window_size: i32, + pub max_speech_duration: c_float, +} + +#[repr(C)] +pub struct TenVadModelConfig { + pub model: *const c_char, + pub threshold: c_float, + pub min_silence_duration: c_float, + pub min_speech_duration: c_float, + pub window_size: i32, + pub max_speech_duration: c_float, +} + +#[repr(C)] +pub struct VadModelConfig { + pub silero_vad: SileroVadModelConfig, + pub sample_rate: i32, + pub num_threads: i32, + pub provider: *const c_char, + pub debug: i32, + pub ten_vad: TenVadModelConfig, +} + +#[repr(C)] +pub struct CircularBuffer { + _private: [u8; 0], +} + +#[repr(C)] +pub struct SpeechSegment { + pub start: i32, + pub samples: *mut f32, + pub n: i32, +} + +#[repr(C)] +pub struct VoiceActivityDetector { + _private: [u8; 0], +} + +extern "C" { + pub fn SherpaOnnxCreateCircularBuffer(capacity: i32) -> *const CircularBuffer; + pub fn SherpaOnnxDestroyCircularBuffer(buffer: *const CircularBuffer); + pub fn SherpaOnnxCircularBufferPush(buffer: *const CircularBuffer, p: *const f32, n: i32); + pub fn SherpaOnnxCircularBufferGet( + buffer: *const CircularBuffer, + start_index: i32, + n: i32, + ) -> *const f32; + pub fn SherpaOnnxCircularBufferFree(p: *const f32); + pub fn SherpaOnnxCircularBufferPop(buffer: *const CircularBuffer, n: i32); + pub fn SherpaOnnxCircularBufferSize(buffer: *const CircularBuffer) -> i32; + pub fn SherpaOnnxCircularBufferHead(buffer: *const CircularBuffer) -> i32; + pub fn SherpaOnnxCircularBufferReset(buffer: *const CircularBuffer); + + pub fn SherpaOnnxCreateVoiceActivityDetector( + config: *const VadModelConfig, + buffer_size_in_seconds: c_float, + ) -> *const VoiceActivityDetector; + pub fn SherpaOnnxDestroyVoiceActivityDetector(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorAcceptWaveform( + p: *const VoiceActivityDetector, + samples: *const f32, + n: i32, + ); + pub fn SherpaOnnxVoiceActivityDetectorEmpty(p: *const VoiceActivityDetector) -> i32; + pub fn SherpaOnnxVoiceActivityDetectorDetected(p: *const VoiceActivityDetector) -> i32; + pub fn SherpaOnnxVoiceActivityDetectorPop(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorClear(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorFront( + p: *const VoiceActivityDetector, + ) -> *const SpeechSegment; + pub fn SherpaOnnxDestroySpeechSegment(p: *const SpeechSegment); + pub fn SherpaOnnxVoiceActivityDetectorReset(p: *const VoiceActivityDetector); + pub fn SherpaOnnxVoiceActivityDetectorFlush(p: *const VoiceActivityDetector); +} diff --git a/crates/sherpa-onnx-sys/src/wave.rs b/crates/sherpa-onnx-sys/src/wave.rs new file mode 100644 index 0000000000..c3c8f860fe --- /dev/null +++ b/crates/sherpa-onnx-sys/src/wave.rs @@ -0,0 +1,30 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use std::os::raw::c_char; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SherpaOnnxWave { + /// Samples normalized to [-1, 1] + pub samples: *const f32, + pub sample_rate: i32, + pub num_samples: i32, +} + +extern "C" { + /// Read a WAV file. Returns NULL on error. + pub fn SherpaOnnxReadWave(filename: *const c_char) -> *const SherpaOnnxWave; + + /// Free memory allocated by SherpaOnnxReadWave + pub fn SherpaOnnxFreeWave(wave: *const SherpaOnnxWave); + + /// Write a WAV file. Returns 1 on success, 0 on failure. + pub fn SherpaOnnxWriteWave( + samples: *const f32, + n: i32, + sample_rate: i32, + filename: *const c_char, + ) -> i32; +} diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 66553ef595..d0f308e6fc 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -75,6 +75,20 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if 1.0.4", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -694,6 +708,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -718,6 +738,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bip39" version = "2.2.2" @@ -1023,6 +1049,7 @@ dependencies = [ "buzz-media", "buzz-persona", "buzz-sdk", + "buzz-voice", "bytes", "bzip2 0.6.1", "chrono", @@ -1143,6 +1170,21 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-voice" +version = "0.1.0" +dependencies = [ + "ort", + "ort-sys", + "rand 0.10.2", + "regex", + "sentencepiece-model", + "serde", + "serde_json", + "sherpa-onnx", + "tokenizers", +] + [[package]] name = "by_address" version = "1.2.1" @@ -1545,6 +1587,7 @@ dependencies = [ "itoa", "rustversion", "ryu", + "serde", "static_assertions", ] @@ -1796,6 +1839,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.20" @@ -2120,6 +2173,15 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dasp_sample" version = "0.11.0" @@ -2617,6 +2679,12 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "euclid" version = "0.22.14" @@ -2675,6 +2743,17 @@ dependencies = [ "regex", ] +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fast-srgb8" version = "1.0.0" @@ -4746,6 +4825,39 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "logos" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "syn 2.0.118", +] + +[[package]] +name = "logos-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" +dependencies = [ + "logos-codegen", +] + [[package]] name = "loom" version = "0.7.2" @@ -4855,6 +4967,22 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + [[package]] name = "markup5ever" version = "0.38.0" @@ -4881,6 +5009,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-async" version = "0.2.11" @@ -4980,7 +5118,7 @@ dependencies = [ "mesh-llm-types", "model-artifact", "nostr-sdk", - "prost", + "prost 0.14.4", "rand 0.10.2", "rustls", "serde", @@ -5119,7 +5257,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", - "prost", + "prost 0.14.4", "rand 0.10.2", "regex-lite", "reqwest 0.12.28", @@ -5207,8 +5345,8 @@ source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=f455d493a2ae82baf2a32 dependencies = [ "anyhow", "async-trait", - "prost", - "prost-build", + "prost 0.14.4", + "prost-build 0.14.4", "protoc-bin-vendored", "rmcp", "schemars 1.2.1", @@ -5243,7 +5381,7 @@ dependencies = [ "anyhow", "hex", "iroh", - "prost", + "prost 0.14.4", "serde_json", "sha2 0.10.9", ] @@ -5358,6 +5496,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if 1.0.4", + "miette-derive", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "mime" version = "0.3.17" @@ -5503,6 +5663,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "more-asserts" version = "0.3.1" @@ -5630,6 +5812,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.9.0" @@ -6611,7 +6808,7 @@ dependencies = [ "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", - "prost", + "prost 0.14.4", "reqwest 0.12.28", "thiserror 2.0.18", ] @@ -6626,7 +6823,7 @@ dependencies = [ "const-hex", "opentelemetry", "opentelemetry_sdk", - "prost", + "prost 0.14.4", "serde", "serde_json", "tonic", @@ -6692,6 +6889,24 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" + [[package]] name = "os_pipe" version = "1.2.3" @@ -6919,6 +7134,16 @@ dependencies = [ "pest", ] +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", + "indexmap 2.14.0", +] + [[package]] name = "petgraph" version = "0.8.3" @@ -7183,6 +7408,15 @@ dependencies = [ "serde", ] +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "portmapper" version = "0.19.1" @@ -7393,6 +7627,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.13.5", +] + [[package]] name = "prost" version = "0.14.4" @@ -7400,7 +7644,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.14.4", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph 0.7.1", + "prettyplease", + "prost 0.13.5", + "prost-types 0.13.5", + "regex", + "syn 2.0.118", + "tempfile", ] [[package]] @@ -7413,15 +7677,28 @@ dependencies = [ "itertools", "log", "multimap", - "petgraph", + "petgraph 0.8.3", "prettyplease", - "prost", - "prost-types", + "prost 0.14.4", + "prost-types 0.14.4", "regex", "syn 2.0.118", "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "prost-derive" version = "0.14.4" @@ -7435,13 +7712,35 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "prost-reflect" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5edd582b62f5cde844716e66d92565d7faf7ab1445c8cebce6e00fba83ddb2" +dependencies = [ + "logos", + "miette", + "once_cell", + "prost 0.13.5", + "prost-types 0.13.5", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost 0.13.5", +] + [[package]] name = "prost-types" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ - "prost", + "prost 0.14.4", ] [[package]] @@ -7508,6 +7807,33 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" +[[package]] +name = "protox" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f352af331bf637b8ecc720f7c87bf903d2571fa2e14a66e9b2558846864b54a" +dependencies = [ + "bytes", + "miette", + "prost 0.13.5", + "prost-reflect", + "prost-types 0.13.5", + "protox-parse", + "thiserror 1.0.69", +] + +[[package]] +name = "protox-parse" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a462d115462c080ae000c29a47f0b3985737e5d3a995fcdbcaa5c782068dde" +dependencies = [ + "logos", + "miette", + "prost-types 0.13.5", + "thiserror 1.0.69", +] + [[package]] name = "pxfm" version = "0.1.30" @@ -7756,7 +8082,7 @@ dependencies = [ "thiserror 2.0.18", "unicode-segmentation", "unicode-truncate", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -7819,7 +8145,7 @@ dependencies = [ "strum", "time", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -7828,6 +8154,43 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "realfft" version = "3.5.0" @@ -8640,6 +9003,18 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" +[[package]] +name = "sentencepiece-model" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b87bf750a8322c3236d7aa63c1f4a6862187d00d2d8b038e1dfe263bfe43ec" +dependencies = [ + "miette", + "prost 0.13.5", + "prost-build 0.13.5", + "protox", +] + [[package]] name = "serde" version = "1.0.228" @@ -9072,8 +9447,8 @@ name = "skippy-protocol" version = "0.73.1" source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=f455d493a2ae82baf2a326e2d0fda351433b4b30#f455d493a2ae82baf2a326e2d0fda351433b4b30" dependencies = [ - "prost", - "prost-build", + "prost 0.14.4", + "prost-build 0.14.4", "protoc-bin-vendored", "serde", ] @@ -9252,6 +9627,18 @@ dependencies = [ "der", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + [[package]] name = "sse-stream" version = "0.2.4" @@ -9633,7 +10020,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce91f2f0ec87dff7e6bcbbeb267439aa1188703003c6055193c821487400432" dependencies = [ - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -10208,7 +10595,7 @@ dependencies = [ "anyhow", "base64 0.22.1", "bitflags 2.13.0", - "fancy-regex", + "fancy-regex 0.11.0", "filedescriptor", "finl_unicode", "fixedbitset 0.4.2", @@ -10371,6 +10758,39 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str 0.9.1", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex 0.14.0", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "paste", + "rand 0.9.4", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.52.3" @@ -10704,7 +11124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", - "prost", + "prost 0.14.4", "tonic", ] @@ -10885,7 +11305,7 @@ checksum = "b8765b90061cba6c22b5831f675da109ae5561588290f9fa2317adab2714d5a6" dependencies = [ "memchr", "nom 8.0.0", - "petgraph", + "petgraph 0.8.3", ] [[package]] @@ -11056,6 +11476,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -11070,9 +11499,15 @@ checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5" dependencies = [ "itertools", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.2" @@ -11085,6 +11520,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "universal-hash" version = "0.5.1" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 324218a49d..1cafc80f72 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -89,6 +89,7 @@ buzz_core_pkg = { package = "buzz-core", path = "../../crates/buzz-core" } buzz_persona_pkg = { package = "buzz-persona", path = "../../crates/buzz-persona" } buzz_sdk_pkg = { package = "buzz-sdk", path = "../../crates/buzz-sdk" } buzz_agent_pkg = { package = "buzz-agent", path = "../../crates/buzz-agent" } +buzz_voice_pkg = { package = "buzz-voice", path = "../../crates/buzz-voice" } iroh = { version = "1.0.2", optional = true } mesh-llm-sdk = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-sdk", default-features = false, features = ["client", "serving"], optional = true } mesh-llm-host-runtime = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f455d493a2ae82baf2a326e2d0fda351433b4b30", package = "mesh-llm-host-runtime", default-features = false, features = ["dynamic-native-runtime"], optional = true } diff --git a/desktop/src-tauri/examples/pocket_quality_ab.rs b/desktop/src-tauri/examples/pocket_quality_ab.rs index 0c31f1c910..829a30440c 100644 --- a/desktop/src-tauri/examples/pocket_quality_ab.rs +++ b/desktop/src-tauri/examples/pocket_quality_ab.rs @@ -15,12 +15,6 @@ //! uninterrupted idle is a valid post-idle observation. Run each 5/15-minute //! item as a separate process. -// Importing the production module also brings in runtime-only helpers that this -// standalone corpus generator deliberately does not call. -#![allow(dead_code)] - -#[path = "../src/huddle/pocket.rs"] -mod production_pocket; #[path = "../src/huddle/preprocessing.rs"] mod production_preprocessing; @@ -33,7 +27,7 @@ use serde::Serialize; use sha2::{Digest, Sha256}; use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; -use production_pocket::{prepare_pocket_prompt, SAMPLE_RATE}; +use buzz_voice_pkg::pocket::{prepare_pocket_prompt, SAMPLE_RATE}; use production_preprocessing::{preprocess_for_tts, split_sentences}; const NUM_STEPS: i32 = 1; diff --git a/desktop/src-tauri/src/huddle/pocket.rs b/desktop/src-tauri/src/huddle/pocket.rs index ee1faf928a..8e19781eeb 100644 --- a/desktop/src-tauri/src/huddle/pocket.rs +++ b/desktop/src-tauri/src/huddle/pocket.rs @@ -1,654 +1,3 @@ -//! Pocket TTS engine wrapper around sherpa-onnx's `OfflineTts`. -//! -//! Pocket TTS is a small (~473 MB fp32 ONNX) zero-shot voice-cloning TTS -//! model from Kyutai. It runs quickly on CPU via sherpa-onnx, replacing the -//! previous Kokoro-82M engine that also required an espeak-free but -//! lexicon-heavy G2P pipeline (Misaki + CMUdict). -//! -//! Full-precision fp32 sessions, not the ~189 MB int8 quantization we -//! originally shipped: a direct same-runtime A/B (k2-fsa/sherpa-onnx#3172) -//! found the int8 ONNX export audibly degraded output quality, and fp32 -//! "significantly improved quality even at 1 step". -//! -//! ## Attribution -//! -//! - **Model**: Kyutai *Pocket TTS* — Charles, Roebel, et al., 2026. -//! arXiv:2509.06926. Original repository: . -//! Licensed CC-BY-4.0. -//! - **Mimi neural codec**: Kyutai, bundled in the same release. CC-BY-4.0. -//! - **ONNX export**: KevinAHM — -//! . CC-BY-4.0. -//! - **sherpa-onnx repackage**: csukuangfj / k2-fsa — -//! . -//! Repackages KevinAHM's export with the file layout sherpa-onnx's -//! `OfflineTtsPocketModelConfig` expects. CC-BY-4.0. -//! - **Reference voice WAV** (`reference_sample.wav`): the "Mary -//! (f, conversation)" preset from the Kyutai TTS demo -//! (), which maps to `vctk/p333_023_enhanced.wav` -//! in . CC-BY-4.0, base recording -//! from the VCTK corpus, enhanced by ai-coustics. -//! -//! Buzz ships these files unmodified; see the on-disk `MODEL_LICENSE.txt` -//! sidecar written by `huddle::models` during install for the canonical -//! CC-BY-4.0 §3(a)(1) attribution block. -//! -//! ## Engine-module contract (see `huddle::tts`) -//! -//! `pocket.rs` exposes a fixed surface used by `tts.rs`. Mirroring this -//! contract is what lets the TTS pipeline stay engine-agnostic: -//! -//! - `SAMPLE_RATE: u32` — engine output sample rate in Hz. -//! - `DEFAULT_VOICE: &str` — default voice name (without extension). -//! - `VOICE_FILE_EXT: &str` — extension for per-voice files on disk. -//! - `load_text_to_speech(model_dir)` → `Result` -//! - `load_voice_style(path)` → `Result` -//! - `Engine::synth_chunk(&self, text, lang, &VoiceStyle, steps)` -//! → `Result, String>` -//! -//! `lang` and `steps` are accepted for API compatibility with the previous -//! Kokoro engine but are unused — Pocket TTS does its own language ID from -//! the input text and is not a diffusion model (consistency LM, one step). -//! There is no speed knob: sherpa-onnx's `GenerationConfig.speed` is only -//! read by some model families (vits), never by the Pocket impl -//! (`offline-tts-pocket-impl.h` — zero references), and upstream pocket-tts -//! has no speed parameter either. +//! Compatibility re-export for the shared Pocket TTS engine. -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use sherpa_onnx::{GenerationConfig, OfflineTts, OfflineTtsConfig, Wave}; - -// ── Engine-module contract: public consts ───────────────────────────────────── - -/// Pocket TTS emits 24 kHz mono PCM. Matches the previous Kokoro output rate, -/// so the rodio sink and inter-sentence silence buffer in `tts.rs` remain valid. -pub const SAMPLE_RATE: u32 = 24_000; - -/// Name (without extension) of the bundled reference voice. The model directory -/// is expected to contain `.` after install. -pub const DEFAULT_VOICE: &str = "reference_sample"; - -/// Voice files for Pocket TTS are reference audio (WAV). Distinct from the -/// Kokoro `.bin` style vectors — the model conditions on raw waveform samples, -/// not a precomputed embedding, so the extension change is honest. -pub const VOICE_FILE_EXT: &str = "wav"; - -// ── Tuning ──────────────────────────────────────────────────────────────────── - -/// Single-threaded ONNX execution for predictable CPU contention with the STT -/// pipeline. Matches `STT_NUM_THREADS` in `stt.rs`; raise only if a benchmark -/// argues for it. -const TTS_NUM_THREADS: i32 = 1; - -/// LRU cache size for cloned voice embeddings inside the sherpa-onnx engine. -/// We bind to one voice per pipeline today, but the upstream example uses 16 -/// and the cost is negligible — keep room for future multi-voice support. -const VOICE_EMBEDDING_CACHE_CAPACITY: i32 = 16; - -/// Pocket TTS is a consistency-based LM. Generation quality saturates at one -/// denoising step — the upstream `GenerationConfig` default of 5 multiplies -/// synthesis time by ~5× with no audible benefit on this model. -const SYNTH_NUM_STEPS: i32 = 1; - -/// Leave the generated audio's silences untouched (1.0 is the identity). -/// -/// sherpa-onnx's `ScaleSilence` (`offline-tts.cc`) is *not* pre/post padding -/// control: it finds every interior silence run ≥ 0.2 s (|s| ≤ 0.01) and -/// multiplies its length by this factor. The previous value of 0.0 — set -/// under the mistaken belief it disabled lead-in/lead-out padding — deleted -/// every natural pause inside an utterance: clause breaks, breaths, the gap -/// after a comma. Words slammed together and endings cut abruptly. The -/// reference Pocket TTS pipeline does not post-process silence at all; -/// 1.0 restores parity. -const SYNTH_SILENCE_SCALE: f32 = 1.0; - -/// sherpa-onnx upstream default for `max_frames` (LM steps), in -/// `offline-tts-pocket-impl.h:Generate`. 500 steps ≈ 40 s of audio at the -/// Mimi 12.5 Hz frame rate. Referenced only by the regression test below; -/// production code path never raises (or even reads) this value — we just -/// leave sherpa-onnx's own default in place by not setting the override. -#[cfg(test)] -const SHERPA_ONNX_MAX_FRAMES_DEFAULT: i32 = 500; - -/// Tight `max_frames` we ask for on short, padded prompts to bound the -/// original "monster breathing" runaway. 100 LM steps ≈ 8 s of audio — -/// roomy for any one-to-four-word utterance the user is likely to elicit -/// while still well short of the 40 s upstream default. Chosen with slack so -/// we never *truncate* a legitimate short reply. -const SHORT_PROMPT_MAX_FRAMES: i32 = 100; - -/// Word-count threshold (inclusive) below which we pad the prompt with -/// leading spaces and cap `max_frames` tighter than the upstream default. -/// Matches upstream `pocket_tts.models.tts_model.prepare_text_prompt`. Above -/// this threshold we leave sherpa-onnx's own defaults in place — overriding -/// them caused the "first 'yep' is just static" regression seen on -/// 2026-05-18, where dropping `frames_after_eos` below the upstream default -/// of 3 clipped the leading audio of multi-clause sentences. -const SHORT_PROMPT_WORD_THRESHOLD: usize = 4; - -/// Number of leading spaces prepended to short prompts. The upstream Python -/// uses exactly 8 — keep parity rather than tuning blindly. -/// -/// This is upstream's *only* mitigation for the FlowLM cold-start smear on -/// short utterances (kyutai-labs/pocket-tts #91, #70): the autoregressive -/// generation has a 2–3 step "settle" period where the first phoneme can be -/// smeared. A previous revision added a sacrificial `". . "` prefix plus an -/// amplitude-threshold trim to strip the rendered prefix from the output — -/// but the trim's absolute threshold (0.02 against raw peaks of ~0.076) sat -/// in soft-onset territory and could eat real word starts, and its tuning -/// was calibrated against `silence_scale = 0.0` audio. Deleted in favour of -/// upstream parity: accept the occasional smeared first syllable rather -/// than risk trimming real speech. -const SHORT_PROMPT_PAD_SPACES: usize = 8; - -/// sherpa-onnx's documented `frames_after_eos` default. We deliberately do -/// *not* override this knob — the previous attempt to bump it for short -/// inputs and lower it for long inputs lowered it below the upstream default -/// of 3, which clipped the leading audio of multi-clause sentences (the -/// "first 'yep' is static" regression). The constant exists only for the -/// regression test below. Source: `offline-tts-pocket-impl.h:Generate`. -#[cfg(test)] -const SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT: i32 = 3; - -// ── ONNX file names (five Pocket TTS sessions plus two JSON tables) ─────────── - -const FILE_LM_MAIN: &str = "lm_main.onnx"; -const FILE_LM_FLOW: &str = "lm_flow.onnx"; -const FILE_ENCODER: &str = "encoder.onnx"; -const FILE_DECODER: &str = "decoder.onnx"; -const FILE_TEXT_COND: &str = "text_conditioner.onnx"; -const FILE_VOCAB: &str = "vocab.json"; -const FILE_TOKEN_SCORES: &str = "token_scores.json"; - -// ── Voice style ─────────────────────────────────────────────────────────────── - -/// Loaded reference voice — normalised f32 PCM samples plus their sample rate. -/// -/// Pocket TTS takes a reference waveform per generation call (not a -/// precomputed style embedding), so we keep the samples in memory and clone -/// the small `Vec` into each `GenerationConfig` rather than re-reading the -/// WAV from disk on every sentence. -#[derive(Debug, Clone)] -pub struct VoiceStyle { - samples: Vec, - sample_rate: i32, -} - -/// Load a reference voice WAV from disk. -/// -/// Accepts any sample rate sherpa-onnx's `Wave::read` can decode — Pocket TTS -/// resamples internally using `reference_sample_rate`. The bundled -/// `reference_sample.wav` ("Mary" — VCTK p333, enhanced) is 32 kHz mono. -pub fn load_voice_style(path: &Path) -> Result { - let path_str = path - .to_str() - .ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?; - let wave = Wave::read(path_str) - .ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?; - let samples = wave.samples().to_vec(); - if samples.is_empty() { - return Err(format!("voice WAV is empty: {}", path.display())); - } - Ok(VoiceStyle { - samples, - sample_rate: wave.sample_rate(), - }) -} - -// ── Engine ──────────────────────────────────────────────────────────────────── - -/// Pocket TTS engine handle. Cheap to construct (one `OfflineTts::create` -/// call). Owned by the TTS worker thread for the lifetime of a huddle session. -/// -/// `OfflineTts` does not implement `Debug`, so we don't derive it here — the -/// pipeline only needs to move the engine into the worker thread and call -/// `synth_chunk` on it, never to print it. -pub struct PocketTts { - inner: OfflineTts, -} - -/// Build the Pocket TTS engine from the model directory installed by -/// `huddle::models`. Returns `Err` if any expected ONNX or JSON file is -/// missing — readiness is normally enforced by `is_tts_ready` upstream, but -/// the check is repeated here so a manually-modified model dir produces a -/// clear error string instead of an opaque sherpa-onnx `None`. -pub fn load_text_to_speech(model_dir: &str) -> Result { - let dir = PathBuf::from(model_dir); - for name in [ - FILE_LM_MAIN, - FILE_LM_FLOW, - FILE_ENCODER, - FILE_DECODER, - FILE_TEXT_COND, - FILE_VOCAB, - FILE_TOKEN_SCORES, - ] { - let p = dir.join(name); - if !p.is_file() { - return Err(format!("missing Pocket TTS file: {}", p.display())); - } - } - - let to_str = |name: &str| -> String { dir.join(name).to_string_lossy().into_owned() }; - - // Build the config by mutating defaults — mirrors `stt.rs` and stays - // resilient if sherpa-onnx adds unrelated model-family fields. - let mut cfg = OfflineTtsConfig::default(); - cfg.model.pocket.lm_main = Some(to_str(FILE_LM_MAIN)); - cfg.model.pocket.lm_flow = Some(to_str(FILE_LM_FLOW)); - cfg.model.pocket.encoder = Some(to_str(FILE_ENCODER)); - cfg.model.pocket.decoder = Some(to_str(FILE_DECODER)); - cfg.model.pocket.text_conditioner = Some(to_str(FILE_TEXT_COND)); - cfg.model.pocket.vocab_json = Some(to_str(FILE_VOCAB)); - cfg.model.pocket.token_scores_json = Some(to_str(FILE_TOKEN_SCORES)); - cfg.model.pocket.voice_embedding_cache_capacity = VOICE_EMBEDDING_CACHE_CAPACITY; - cfg.model.num_threads = TTS_NUM_THREADS; - // Explicit — defaults are not part of the API contract, and noisy debug - // logging in release builds would be expensive on every synthesized chunk. - cfg.model.debug = false; - - let inner = OfflineTts::create(&cfg) - .ok_or_else(|| "OfflineTts::create returned None for Pocket TTS".to_string())?; - Ok(PocketTts { inner }) -} - -// ── Prompt preparation ──────────────────────────────────────────────────────── - -/// Result of [`prepare_pocket_prompt`]: a synthesizer-ready prompt plus the -/// per-call generation overrides derived from the original text. -/// -/// `None` for either override means "leave sherpa-onnx's documented default -/// in place". The pipeline only sets `max_frames` (and only for short -/// padded inputs) so it can bound the original "monster breathing" runaway -/// without disturbing the rest of the LM sampling envelope. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct PreparedPrompt { - /// Text to hand to `OfflineTts::generate_with_config`. Capitalized, - /// punctuation-terminated, and (for short inputs) left-padded with - /// spaces — upstream's mitigation for the FlowLM cold-start smear. - pub text: String, - /// Value to pass via `GenerationConfig.extra["max_frames"]`, or `None` to - /// keep the upstream default of 500 LM steps. We only override on short - /// padded prompts where we have a tight expectation on output length. - pub max_frames: Option, -} - -/// Mirror of the *text-preparation* half of upstream -/// `pocket_tts.models.tts_model.prepare_text_prompt`. Sherpa-onnx's C++ -/// Pocket TTS impl does not run these preparation steps, so short / -/// unpunctuated / lowercase inputs can trigger up to 40 s of runaway -/// generation when the EOS logit never crosses its threshold. We replicate -/// the upstream Python recipe here: -/// -/// 1. Collapse interior whitespace (already done by `preprocess_for_tts`, but -/// cheap to re-check after sentence splitting). -/// 2. Capitalize the first letter. -/// 3. Append `.` if the text doesn't end in punctuation. -/// 4. If fewer than five words, prepend `SHORT_PROMPT_PAD_SPACES` spaces -/// (upstream's cold-start mitigation — see the constant's docstring) and -/// return a tight [`SHORT_PROMPT_MAX_FRAMES`] cap so the LM can't run -/// away if EOS still doesn't fire. -/// -/// We do **not** override `frames_after_eos` — sherpa-onnx's default of 3 -/// is what we want. An earlier version set it to 1 on long inputs, which -/// clipped the leading audio of multi-clause sentences ("first 'yep' is -/// just static" regression). Tests `prepare_prompt_never_lowers_frames_…` -/// lock this in. -/// -/// Returns `None` only if the input is empty after trimming — caller should -/// skip synthesis in that case. -pub(crate) fn prepare_pocket_prompt(input: &str) -> Option { - let trimmed = input.trim(); - if trimmed.is_empty() { - return None; - } - - // Collapse stray double-spaces / embedded newlines that may slip past - // `preprocess_for_tts` when sentences are spliced back together. - let mut cleaned = String::with_capacity(trimmed.len()); - let mut last_was_space = false; - for ch in trimmed.chars() { - let is_ws = ch.is_whitespace(); - if is_ws { - if !last_was_space { - cleaned.push(' '); - } - last_was_space = true; - } else { - cleaned.push(ch); - last_was_space = false; - } - } - - // Capitalize first character. Uses `to_uppercase` (multi-codepoint safe). - let first = cleaned.chars().next().expect("cleaned non-empty above"); - if first.is_lowercase() { - let upper: String = first.to_uppercase().collect(); - let mut iter = cleaned.chars(); - iter.next(); - cleaned = upper + iter.as_str(); - } - - // Ensure terminal punctuation. Anything not in `.!?;:,` gets a period. - // The upstream Python only checks `isalnum` → period, but for our agent - // text we already may end in `!` `?` `.` etc. — treat any of those as OK. - let last = cleaned - .chars() - .next_back() - .expect("cleaned non-empty above"); - if !matches!(last, '.' | '!' | '?' | ';' | ':' | ',') { - cleaned.push('.'); - } - - // Word count of the *cleaned but not padded* text — padding is whitespace - // only and would just lie to the threshold check below. - let word_count = cleaned.split_whitespace().count(); - - let (final_text, max_frames) = if word_count <= SHORT_PROMPT_WORD_THRESHOLD { - let mut padded = String::with_capacity(cleaned.len() + SHORT_PROMPT_PAD_SPACES); - for _ in 0..SHORT_PROMPT_PAD_SPACES { - padded.push(' '); - } - padded.push_str(&cleaned); - (padded, Some(SHORT_PROMPT_MAX_FRAMES)) - } else { - // For everything ≥5 words, fall back to upstream defaults. Overriding - // these is what caused the "first 'yep' is static" regression — the - // upstream LM has been tuned for `frames_after_eos = 3` and - // `max_frames = 500`, and there's no clear win in second-guessing. - (cleaned, None) - }; - - Some(PreparedPrompt { - text: final_text, - max_frames, - }) -} - -/// Build the `GenerationConfig.extra` HashMap from a [`PreparedPrompt`]. -/// -/// Centralised so the regression test below can assert that we **never** -/// emit a `frames_after_eos` override — the previous attempt to override -/// that knob (setting it to 1 for ≥5-word inputs) clipped the leading -/// audio of multi-clause sentences (the "first 'yep' is static" bug on -/// 2026-05-18). The upstream sherpa-onnx default of 3 is what we want, and -/// the right way to keep it is to not set it at all. -fn build_generation_extra(prepared: &PreparedPrompt) -> Option> { - prepared.max_frames.map(|mf| { - let mut h: HashMap = HashMap::with_capacity(1); - h.insert("max_frames".to_string(), serde_json::Value::from(mf)); - h - }) -} - -impl PocketTts { - /// Synthesise `text` with the given reference voice. - /// - /// `_lang` and `_steps` are accepted for API compatibility with the - /// previous Kokoro engine. Pocket TTS infers language from the input text - /// directly and is a one-step consistency model. Returns an empty buffer - /// for whitespace-only input. - pub fn synth_chunk( - &self, - text: &str, - _lang: &str, - style: &VoiceStyle, - _steps: usize, - ) -> Result, String> { - // Mirror upstream pocket-tts prompt prep — without this short or - // unpunctuated inputs can cause the LM's EOS logit to never trip, - // producing up to 40 s of "monster breathing" garbage on the first - // utterance. See `prepare_pocket_prompt` for the full recipe. - let prepared = match prepare_pocket_prompt(text) { - Some(p) => p, - None => return Ok(Vec::new()), - }; - - // Per-call generation hints sherpa-onnx forwards to - // `offline-tts-pocket-impl.h`. We only override `max_frames`, and - // only for short padded prompts where we have a tight expectation - // on output length — that bounds the original runaway without - // disturbing the rest of the LM sampling envelope. See - // `prepare_pocket_prompt` docs for the regression history. - let extra = build_generation_extra(&prepared); - - let cfg = GenerationConfig { - num_steps: SYNTH_NUM_STEPS, - silence_scale: SYNTH_SILENCE_SCALE, - reference_audio: Some(style.samples.clone()), - reference_sample_rate: style.sample_rate, - extra, - // `speed` stays at its default: the Pocket impl never reads it - // (see the engine-contract note in the module docs). - ..Default::default() - }; - - // No progress callback — synthesis is fast enough that returning the - // whole buffer at once keeps the lookahead pipelining in `tts.rs` - // simple. `None:: bool>` pins the callback type for the - // `generate_with_config` generic parameter. - let audio = self - .inner - .generate_with_config(&prepared.text, &cfg, None:: bool>) - .ok_or_else(|| { - format!( - "Pocket TTS synthesis failed for text ({} chars)", - prepared.text.len() - ) - })?; - - let sample_rate = audio.sample_rate(); - if sample_rate != SAMPLE_RATE as i32 { - eprintln!( - "buzz-desktop: Pocket TTS returned unexpected sample rate {sample_rate}Hz \ - (expected {SAMPLE_RATE}Hz); playback speed may be wrong" - ); - } - - Ok(audio.samples().to_vec()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // ── prepare_pocket_prompt ──────────────────────────────────────────────── - - #[test] - fn prepare_prompt_returns_none_for_empty_input() { - assert!(prepare_pocket_prompt("").is_none()); - assert!(prepare_pocket_prompt(" ").is_none()); - assert!(prepare_pocket_prompt("\n\t ").is_none()); - } - - /// Helper: the exact leading sequence prepended to every short prompt — - /// 8 spaces of padding (upstream's cold-start mitigation). - /// Centralising this keeps the assertions readable. - fn short_prefix() -> String { - " ".repeat(SHORT_PROMPT_PAD_SPACES) - } - - #[test] - fn prepare_prompt_pads_and_capitalizes_one_word() { - // The "yep" case Tyler hit in production — bare lowercase one-word - // utterance with no punctuation. Must be padded with the short-prompt - // space pad, capitalized, terminated, with a tight `max_frames` cap - // to bound runaway gen. - let out = prepare_pocket_prompt("yep").expect("non-empty"); - assert_eq!(out.text, format!("{}Yep.", short_prefix())); - assert_eq!(out.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - const { - assert!( - SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT, - "short cap must be tighter than the upstream default" - ); - } - } - - #[test] - fn prepare_prompt_preserves_existing_punctuation() { - let out = prepare_pocket_prompt("yes!").expect("non-empty"); - assert_eq!(out.text, format!("{}Yes!", short_prefix())); // exclamation kept - let out = prepare_pocket_prompt("really?").expect("non-empty"); - assert_eq!(out.text, format!("{}Really?", short_prefix())); - } - - #[test] - fn prepare_prompt_threshold_is_inclusive_at_four_words() { - // 4 words = short (padded + tight max_frames); 5 words = long - // (no padding, no overrides — upstream defaults stand). - let four = prepare_pocket_prompt("one two three four").expect("non-empty"); - assert_eq!( - four.text, - format!("{}One two three four.", short_prefix()), - "four-word input should get exactly the space pad" - ); - assert_eq!(four.max_frames, Some(SHORT_PROMPT_MAX_FRAMES)); - - let five = prepare_pocket_prompt("one two three four five").expect("non-empty"); - assert!( - !five.text.starts_with(' '), - "five-word input should NOT be padded" - ); - assert_eq!( - five.max_frames, None, - "long inputs must leave sherpa-onnx's max_frames default in place" - ); - } - - #[test] - fn prepare_prompt_does_not_pad_long_text() { - let long = "This is a longer sentence that the model should handle just fine."; - let out = prepare_pocket_prompt(long).expect("non-empty"); - assert!(!out.text.starts_with(' ')); - assert_eq!(out.max_frames, None); - assert!(out.text.ends_with('.')); - } - - #[test] - fn prepare_prompt_collapses_whitespace() { - let out = prepare_pocket_prompt("Hello world\n\nfriend").expect("non-empty"); - // 3 words → short → padded. Interior whitespace collapsed. - assert_eq!(out.text, format!("{}Hello world friend.", short_prefix())); - } - - #[test] - fn prepare_prompt_does_not_double_capitalize_already_uppercase() { - let out = prepare_pocket_prompt("HELLO there").expect("non-empty"); - assert_eq!(out.text, format!("{}HELLO there.", short_prefix())); - } - - #[test] - fn prepare_prompt_handles_non_ascii_first_letter() { - // Cyrillic lowercase 'д' → uppercase 'Д'. Must not panic / produce - // mojibake. - let out = prepare_pocket_prompt("дa").expect("non-empty"); - assert!(out.text.contains("Дa.")); - } - - /// REGRESSION GUARD: short prompts must receive *only* whitespace - /// padding — no sacrificial text. A previous revision prepended a - /// `". . "` cold-start absorber and trimmed the rendered audio back out - /// with an amplitude threshold that could eat soft word onsets. If - /// non-whitespace ever reappears in the pad, the synth output will - /// contain audio for text the user never wrote. - #[test] - fn prepare_prompt_pad_is_whitespace_only() { - let out = prepare_pocket_prompt("I'm happy.").expect("non-empty"); - let pad_len = out.text.len() - "I'm happy.".len(); - assert!( - out.text[..pad_len].chars().all(|c| c == ' '), - "short-prompt pad must be spaces only, got {:?}", - &out.text[..pad_len] - ); - assert_eq!(out.text, format!("{}I'm happy.", short_prefix())); - } - - // ── build_generation_extra ─────────────────────────────────────────────── - // - // These tests pin down a behaviour we've now regressed twice on: - // 1) Not padding/punctuating short inputs → 40 s of "monster breathing" - // (pre-773a2a1). - // 2) Setting `frames_after_eos = 1` on long inputs → clipped leading - // audio of multi-clause sentences, e.g. "Yep, I can hear you. …" - // came out as a static burst (the 773a2a1 regression Tyler hit on - // 2026-05-18 ~14:30 UTC). - // - // The contract we enforce going forward: we **only** override - // `max_frames`, and only for ≤4-word inputs. Every other knob is left - // at sherpa-onnx's documented default (notably `frames_after_eos = 3`). - - #[test] - fn build_extra_short_prompt_sets_only_max_frames() { - let prepared = prepare_pocket_prompt("yep").expect("non-empty"); - let extra = build_generation_extra(&prepared).expect("short prompts get extra"); - // Exactly one key — `max_frames` — and nothing else. - assert_eq!(extra.len(), 1, "extra has unexpected keys: {extra:?}"); - assert_eq!( - extra.get("max_frames"), - Some(&serde_json::Value::from(SHORT_PROMPT_MAX_FRAMES)) - ); - assert!( - !extra.contains_key("frames_after_eos"), - "frames_after_eos must never be set — upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT} is what we want" - ); - } - - #[test] - fn build_extra_long_prompt_is_none() { - // ≥5 words: no extras at all. This is the key fix for the "first - // 'yep' in 'Yep, I can hear you. …' is static" regression — we - // were previously forcing `frames_after_eos = 1` on this path. - let prepared = prepare_pocket_prompt("Yep, I can hear you.").expect("non-empty"); - assert_eq!( - build_generation_extra(&prepared), - None, - "long prompts must not override any LM knob" - ); - } - - #[test] - fn build_extra_never_lowers_frames_after_eos_for_any_word_count() { - // Sweep a range of prompt lengths and assert the `extra` map (when - // present) never carries a `frames_after_eos` override that's lower - // than the upstream sherpa-onnx default. Implemented as a structural - // check — we just never set the key — but worth a property test in - // case someone reintroduces the override in the future. - let prompts: &[&str] = &[ - "hi", - "hi there", - "yes please", - "one two three four", - "one two three four five", - "a slightly longer reply, hopefully fine", - "This is a multi-clause sentence. It has two parts.", - "really really really really really long prompt with lots of words just to be sure", - ]; - for &p in prompts { - let prepared = prepare_pocket_prompt(p).expect("non-empty"); - if let Some(extra) = build_generation_extra(&prepared) { - if let Some(v) = extra.get("frames_after_eos") { - let n = v.as_i64().expect("frames_after_eos should be int"); - assert!( - n >= SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT as i64, - "prompt {p:?} set frames_after_eos={n}, below upstream default of {SHERPA_ONNX_FRAMES_AFTER_EOS_DEFAULT}" - ); - } - } - } - } - - #[test] - fn short_prompt_max_frames_is_below_upstream_default() { - // Sanity: the override only ever *lowers* the cap, never raises it. - const { - assert!(SHORT_PROMPT_MAX_FRAMES < SHERPA_ONNX_MAX_FRAMES_DEFAULT); - } - // …and is still large enough for a one-to-four-word reply. At Mimi's - // 12.5 Hz frame rate, 100 frames = 8 s, which is roomy. - const { - assert!(SHORT_PROMPT_MAX_FRAMES >= 50, "would risk truncation"); - } - } -} +pub use buzz_voice_pkg::pocket::*; diff --git a/desktop/src-tauri/src/huddle/preprocessing.rs b/desktop/src-tauri/src/huddle/preprocessing.rs index ce85e3145e..ccc3fa0668 100644 --- a/desktop/src-tauri/src/huddle/preprocessing.rs +++ b/desktop/src-tauri/src/huddle/preprocessing.rs @@ -1,667 +1,6 @@ -//! Text preprocessing for TTS output. -//! -//! Mental model: -//! -//! ```text -//! raw agent text -//! → strip fenced code blocks → "code block omitted" -//! → strip inline code → bare text -//! → strip URLs → "link omitted" -//! → strip markdown markers → plain text -//! → strip emoji → (removed) -//! → numbers → words → "forty two" -//! → collapse whitespace → clean string -//! ``` -//! -//! Also provides `split_sentences` — the single sentence-boundary splitter used -//! by both the TTS batching pipeline and the Supertonic text chunker. +//! Compatibility exports for shared TTS preparation. -use regex::Regex; -use std::sync::LazyLock; - -// ── Sentence splitting ──────────────────────────────────────────────────────── - -/// Regex: a sentence-ending punctuation mark followed by whitespace. -static RE_SENTENCE_BOUNDARY: LazyLock = LazyLock::new(|| Regex::new(r"([.!?])\s+").unwrap()); - -/// Common abbreviations that end with a period but are NOT sentence boundaries. -const ABBREVIATIONS: &[&str] = &[ - "Dr.", "Mr.", "Mrs.", "Ms.", "Prof.", "Sr.", "Jr.", "St.", "Ave.", "Rd.", "Blvd.", "Dept.", - "Inc.", "Ltd.", "Co.", "Corp.", "etc.", "vs.", "i.e.", "e.g.", "Ph.D.", -]; - -/// Split text into sentence-sized chunks. -/// -/// Combines regex-based boundary detection with: -/// - Abbreviation awareness (`Dr.`, `Mr.`, etc. don't split) -/// - Digit-before-period check (avoids splitting `1.` `2.` numbered lists) -/// - `\n` and `—` treated as sentence breaks -/// -/// Returns non-empty, trimmed strings. -pub fn split_sentences(text: &str) -> Vec { - // First, split on newlines and em-dashes to get coarse segments. - let coarse: Vec<&str> = text.split(['\n', '—']).collect(); - - let mut sentences = Vec::new(); - - for segment in coarse { - let segment = segment.trim(); - if segment.is_empty() { - continue; - } - // Within each segment, split on sentence-ending punctuation. - let matches: Vec<_> = RE_SENTENCE_BOUNDARY.find_iter(segment).collect(); - if matches.is_empty() { - sentences.push(segment.to_string()); - continue; - } - - let mut last_end = 0usize; - for m in &matches { - let before = &segment[last_end..m.start()]; - let punc_char = &segment[m.start()..m.start() + 1]; - - // Skip if this looks like an abbreviation. - let combined = format!("{}{}", before.trim(), punc_char); - let is_abbrev = ABBREVIATIONS.iter().any(|a| combined.ends_with(a)); - - // Skip if the character before the period is a digit (numbered list). - let is_digit_period = punc_char == "." - && !before.is_empty() - && before.ends_with(|c: char| c.is_ascii_digit()); - - if !is_abbrev && !is_digit_period { - let piece = segment[last_end..m.end()].trim(); - if !piece.is_empty() { - sentences.push(piece.to_string()); - } - last_end = m.end(); - } - } - - if last_end < segment.len() { - let tail = segment[last_end..].trim(); - if !tail.is_empty() { - sentences.push(tail.to_string()); - } - } - } - - if sentences.is_empty() { - vec![text.to_string()] - } else { - sentences - } -} - -// ── Public API ──────────────────────────────────────────────────────────────── - -/// Prepare `text` for TTS synthesis. -/// -/// Applies in order: -/// 1. Fenced code blocks → "code block omitted" -/// 2. Inline code → bare content (backticks stripped) -/// 3. URLs → "link omitted" -/// 4. Markdown bold/italic/underline markers stripped -/// 5. Emoji stripped -/// 6. Numbers → words (integers 0–999, times HH:MM) -/// 7. Excess whitespace collapsed -pub fn preprocess_for_tts(text: &str) -> String { - let s = strip_fenced_code_blocks(text); - let s = strip_inline_code(&s); - let s = strip_urls(&s); - let s = strip_markdown_markers(&s); - let s = strip_emoji(&s); - let s = expand_numbers(&s); - let s = collapse_whitespace(&s); - // Filter trivially short results — ".", ",", etc. would be spoken as - // "period", "comma" by TTS. Agents that have nothing relevant to say - // should not respond at all, but defense-in-depth catches edge cases. - if s.len() <= 1 { - return String::new(); - } - s -} - -// ── Step implementations ────────────────────────────────────────────────────── - -/// Replace fenced code blocks with "code block omitted". -/// -/// Handles both ` ``` ` and `~~~` fences. Multi-line aware. -fn strip_fenced_code_blocks(text: &str) -> String { - let s = replace_fenced(text, "```"); - replace_fenced(&s, "~~~") -} - -fn replace_fenced(text: &str, fence: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut rest = text; - loop { - match rest.find(fence) { - None => { - out.push_str(rest); - break; - } - Some(start) => { - // Everything before the opening fence. - out.push_str(&rest[..start]); - rest = &rest[start + fence.len()..]; - // Skip optional language tag on the same line. - if let Some(nl) = rest.find('\n') { - rest = &rest[nl + 1..]; - } - // Find the closing fence. - match rest.find(fence) { - None => { - // Unclosed fence — treat rest as omitted. - out.push_str(" code block omitted "); - break; - } - Some(end) => { - out.push_str(" code block omitted "); - rest = &rest[end + fence.len()..]; - // Skip trailing newline after closing fence. - if rest.starts_with('\n') { - rest = &rest[1..]; - } - } - } - } - } - } - out -} - -/// Strip backtick-delimited inline code, leaving the inner text. -/// -/// Single-backtick only — triple backtick already handled above. -fn strip_inline_code(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut rest = text; - loop { - match rest.find('`') { - None => { - out.push_str(rest); - break; - } - Some(start) => { - out.push_str(&rest[..start]); - rest = &rest[start + 1..]; - match rest.find('`') { - None => { - // Unclosed — emit as-is. - out.push_str(rest); - break; - } - Some(end) => { - out.push_str(&rest[..end]); - rest = &rest[end + 1..]; - } - } - } - } - } - out -} - -/// Replace http/https URLs with "link omitted". -/// -/// Trailing sentence-ending punctuation (`.`, `!`, `?`) that immediately follows -/// a URL and is at end-of-string or followed by whitespace is preserved so that -/// sentence splitting and TTS prosody are not degraded. -/// -/// Example: `"See https://x.y/z."` → `"See link omitted."` -fn strip_urls(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut rest = text; - loop { - // Find the earliest URL prefix. - let http = rest.find("http://"); - let https = rest.find("https://"); - let url_start = match (http, https) { - (None, None) => { - out.push_str(rest); - break; - } - (Some(a), None) => a, - (None, Some(b)) => b, - (Some(a), Some(b)) => a.min(b), - }; - out.push_str(&rest[..url_start]); - rest = &rest[url_start..]; - // Consume until whitespace or structural delimiter. - let url_end = rest - .find(|c: char| c.is_whitespace() || c == ')' || c == ']' || c == '"' || c == '\'') - .unwrap_or(rest.len()); - let url_token = &rest[..url_end]; - rest = &rest[url_end..]; - - // Check if the URL token ends with sentence-ending punctuation that - // belongs to the surrounding sentence rather than the URL itself. - // A trailing `.`, `!`, or `?` is preserved when it is at end-of-string - // or followed by whitespace (i.e. it is a sentence boundary). - let trailing_punct = if url_token.ends_with(['.', '!', '?']) { - let after = rest; // rest is already past url_end - if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { - // Preserve the trailing punctuation. - &url_token[url_token.len() - 1..] - } else { - "" - } - } else { - "" - }; - - out.push_str("link omitted"); - out.push_str(trailing_punct); - } - out -} - -/// Strip `**`, `*`, `__`, `_emphasis_`, `~~` markdown markers. -/// -/// Underscores are only stripped when they wrap a word (`_text_`). -/// Standalone underscores (e.g. `snake_case` identifiers) are preserved. -fn strip_markdown_markers(text: &str) -> String { - // Order matters: strip multi-char markers before single-char. - let s = text.replace("**", ""); - let s = s.replace("__", ""); - let s = s.replace("~~", ""); - let s = s.replace('*', ""); - strip_underscore_emphasis(&s) -} - -/// Strip `_text_` emphasis markers while preserving underscores in identifiers. -/// -/// A `_` is treated as an emphasis delimiter only when it is preceded by -/// whitespace or the start of the string AND followed by a non-whitespace char, -/// or vice-versa for the closing delimiter. -fn strip_underscore_emphasis(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let chars: Vec = text.chars().collect(); - let len = chars.len(); - let mut i = 0; - while i < len { - if chars[i] == '_' { - // Opening delimiter: preceded by whitespace/start, followed by non-whitespace. - let prev_is_boundary = i == 0 || chars[i - 1].is_whitespace(); - let next_is_nonspace = i + 1 < len && !chars[i + 1].is_whitespace(); - if prev_is_boundary && next_is_nonspace { - // Look for a matching closing `_`. - if let Some(close) = (i + 1..len).find(|&j| { - chars[j] == '_' - && !chars[j - 1].is_whitespace() - && (j + 1 >= len - || chars[j + 1].is_whitespace() - || chars[j + 1].is_ascii_punctuation()) - }) { - // Emit the inner text without the delimiters. - for &ch in &chars[i + 1..close] { - out.push(ch); - } - i = close + 1; - continue; - } - } - // Not an emphasis delimiter — emit as-is. - out.push('_'); - } else { - out.push(chars[i]); - } - i += 1; - } - out -} - -/// Strip Unicode emoji (characters in common emoji ranges). -/// -/// Covers the main Emoji block (U+1F300–U+1FAFF) and supplemental ranges. -/// ASCII emoticons like `:)` are left as-is. -fn strip_emoji(text: &str) -> String { - text.chars().filter(|&c| !is_emoji(c)).collect() -} - -#[inline] -fn is_emoji(c: char) -> bool { - matches!(c, - '\u{1F300}'..='\u{1FAFF}' // Misc symbols, emoticons, transport, etc. - | '\u{2600}'..='\u{27BF}' // Misc symbols, dingbats - | '\u{FE00}'..='\u{FE0F}' // Variation selectors - | '\u{1F000}'..='\u{1F02F}'// Mahjong/domino tiles - | '\u{1F0A0}'..='\u{1F0FF}'// Playing cards - | '\u{200D}' // Zero-width joiner (used in emoji sequences) - | '\u{20E3}' // Combining enclosing keycap - ) -} - -/// Expand numbers to spoken words. -/// -/// Handles: -/// - Times: `HH:MM` → "eleven thirty" -/// - Integers 0–999,999 -/// - Leaves other numeric strings (e.g. "3.14", "1000000+") as-is. -fn expand_numbers(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut chars = text.char_indices().peekable(); - - while let Some((i, c)) = chars.next() { - if c.is_ascii_digit() { - // Collect the full token (digits, colon, dots). - let start = i; - let mut end = i + c.len_utf8(); - while let Some(&(j, nc)) = chars.peek() { - if nc.is_ascii_digit() || nc == ':' || nc == '.' { - end = j + nc.len_utf8(); - chars.next(); - } else { - break; - } - } - let token = &text[start..end]; - out.push_str(&expand_numeric_token(token)); - } else { - out.push(c); - } - } - out -} - -fn expand_numeric_token(token: &str) -> String { - // Strip trailing punctuation that the token collector may have included - // (e.g. "11:30." from "at 11:30.") before attempting to parse. - let token = token.trim_end_matches(|c: char| !c.is_ascii_digit()); - - // Time: HH:MM - if let Some(colon) = token.find(':') { - let h = &token[..colon]; - let m = &token[colon + 1..]; - if let (Ok(hh), Ok(mm)) = (h.parse::(), m.parse::()) { - if hh < 24 && mm < 60 { - let hour_word = int_to_words(hh); - let min_word = if mm == 0 { - String::new() - } else if mm < 10 { - // "9:05" → "nine oh five" (not "nine five") - format!(" oh {}", int_to_words(mm)) - } else { - format!(" {}", int_to_words(mm)) - }; - return format!("{}{}", hour_word, min_word); - } - } - // Not a valid time — return as-is. - return token.to_string(); - } - - // Plain integer 0–999,999. - if token.chars().all(|c| c.is_ascii_digit()) { - if let Ok(n) = token.parse::() { - if n <= 999_999 { - return int_to_words(n); - } - } - } - - // Anything else (decimals, millions+) — leave as-is. - token.to_string() -} - -/// Convert an integer 0–999,999 to English words. -fn int_to_words(n: u32) -> String { - const ONES: &[&str] = &[ - "zero", - "one", - "two", - "three", - "four", - "five", - "six", - "seven", - "eight", - "nine", - "ten", - "eleven", - "twelve", - "thirteen", - "fourteen", - "fifteen", - "sixteen", - "seventeen", - "eighteen", - "nineteen", - ]; - const TENS: &[&str] = &[ - "", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety", - ]; - - if n < 20 { - return ONES[n as usize].to_string(); - } - if n < 100 { - let ten = TENS[(n / 10) as usize]; - let one = n % 10; - return if one == 0 { - ten.to_string() - } else { - format!("{} {}", ten, ONES[one as usize]) - }; - } - if n < 1000 { - let hundreds = n / 100; - let remainder = n % 100; - let hundred_word = format!("{} hundred", ONES[hundreds as usize]); - return if remainder == 0 { - hundred_word - } else { - format!("{} {}", hundred_word, int_to_words(remainder)) - }; - } - // 1,000–999,999 - let thousands = n / 1000; - let remainder = n % 1000; - let thousand_word = format!("{} thousand", int_to_words(thousands)); - if remainder == 0 { - thousand_word - } else { - format!("{} {}", thousand_word, int_to_words(remainder)) - } -} - -/// Collapse runs of whitespace (spaces, tabs, newlines) to a single space. -/// Trims leading/trailing whitespace. -fn collapse_whitespace(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut prev_space = true; // Start true to trim leading whitespace. - for c in text.chars() { - if c.is_whitespace() { - if !prev_space { - out.push(' '); - prev_space = true; - } - } else { - out.push(c); - prev_space = false; - } - } - // Trim trailing space. - if out.ends_with(' ') { - out.pop(); - } - out -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn strips_fenced_code_block() { - let input = "Here is some code:\n```rust\nfn main() {}\n```\nDone."; - let out = preprocess_for_tts(input); - assert!(out.contains("code block omitted"), "got: {out}"); - assert!(!out.contains("fn main"), "got: {out}"); - } - - #[test] - fn strips_inline_code() { - let out = preprocess_for_tts("Call `foo()` now."); - assert_eq!(out, "Call foo() now."); - } - - #[test] - fn strips_urls() { - let out = preprocess_for_tts("See https://example.com for details."); - assert!(out.contains("link omitted"), "got: {out}"); - assert!(!out.contains("example.com"), "got: {out}"); - } - - #[test] - fn strips_url_preserves_trailing_period() { - // Trailing `.` at end of sentence must be preserved for sentence splitting. - let out = strip_urls("See https://x.y/z."); - assert_eq!(out, "See link omitted.", "got: {out}"); - } - - #[test] - fn strips_url_preserves_trailing_exclamation() { - let out = strip_urls("Visit https://example.com!"); - assert_eq!(out, "Visit link omitted!", "got: {out}"); - } - - #[test] - fn strips_url_preserves_trailing_question() { - let out = strip_urls("Did you see https://example.com?"); - assert_eq!(out, "Did you see link omitted?", "got: {out}"); - } - - #[test] - fn strips_url_mid_sentence_no_punct_preserved() { - // URL in the middle of a sentence — no trailing punct to preserve. - let out = strip_urls("Check https://example.com for more info."); - assert_eq!(out, "Check link omitted for more info.", "got: {out}"); - } - - #[test] - fn strips_bold_italic() { - let out = preprocess_for_tts("**bold** and *italic* and _under_"); - assert_eq!(out, "bold and italic and under"); - } - - #[test] - fn preserves_standalone_underscores() { - // snake_case identifiers should not be mangled. - let out = preprocess_for_tts("call foo_bar() or baz_qux"); - assert!(out.contains("foo_bar"), "got: {out}"); - assert!(out.contains("baz_qux"), "got: {out}"); - } - - #[test] - fn strips_tilde_fenced_block() { - let input = "Here:\n~~~python\nprint('hi')\n~~~\nDone."; - let out = preprocess_for_tts(input); - assert!(out.contains("code block omitted"), "got: {out}"); - assert!(!out.contains("print"), "got: {out}"); - } - - #[test] - fn expands_integers() { - assert_eq!(preprocess_for_tts("42"), "forty two"); - assert_eq!(preprocess_for_tts("0"), "zero"); - assert_eq!(preprocess_for_tts("11"), "eleven"); - assert_eq!(preprocess_for_tts("100"), "one hundred"); - } - - #[test] - fn expands_thousands() { - assert_eq!(preprocess_for_tts("1000"), "one thousand"); - assert_eq!( - preprocess_for_tts("1234"), - "one thousand two hundred thirty four" - ); - assert_eq!(preprocess_for_tts("10000"), "ten thousand"); - assert_eq!(preprocess_for_tts("100000"), "one hundred thousand"); - assert_eq!( - preprocess_for_tts("999999"), - "nine hundred ninety nine thousand nine hundred ninety nine" - ); - } - - #[test] - fn expands_times() { - assert_eq!(preprocess_for_tts("11:30"), "eleven thirty"); - assert_eq!(preprocess_for_tts("9:00"), "nine"); - assert_eq!(preprocess_for_tts("9:05"), "nine oh five"); - assert_eq!(preprocess_for_tts("10:09"), "ten oh nine"); - } - - #[test] - fn collapses_whitespace() { - let out = preprocess_for_tts(" hello world "); - assert_eq!(out, "hello world"); - } - - #[test] - fn split_sentences_basic() { - let result = split_sentences("Hello world. How are you? I'm fine!"); - assert_eq!(result, vec!["Hello world.", "How are you?", "I'm fine!"]); - } - - #[test] - fn split_sentences_newline_break() { - let result = split_sentences("First line.\nSecond line."); - assert_eq!(result, vec!["First line.", "Second line."]); - } - - #[test] - fn split_sentences_em_dash_break() { - let result = split_sentences("Start here—then continue."); - assert_eq!(result, vec!["Start here", "then continue."]); - } - - #[test] - fn split_sentences_abbreviations() { - let result = split_sentences("Dr. Smith went home. He was tired."); - assert_eq!(result, vec!["Dr. Smith went home.", "He was tired."]); - } - - #[test] - fn split_sentences_numbered_list() { - let result = split_sentences("1. First item. 2. Second item."); - // "1." and "2." should NOT cause a split (digit before period). - assert_eq!(result, vec!["1. First item.", "2. Second item."]); - } - - #[test] - fn split_sentences_single() { - let result = split_sentences("Just one sentence"); - assert_eq!(result, vec!["Just one sentence"]); - } - - #[test] - fn split_sentences_empty() { - let result = split_sentences(""); - assert_eq!(result, vec![""]); - } - - #[test] - fn filters_trivial_responses() { - assert_eq!(preprocess_for_tts("."), ""); - assert_eq!(preprocess_for_tts(","), ""); - assert_eq!(preprocess_for_tts("!"), ""); - assert_eq!(preprocess_for_tts(" "), ""); - assert_eq!(preprocess_for_tts("ok"), "ok"); - } - - #[test] - fn full_pipeline() { - let input = - "**Agent says:** check https://relay.example.com at 11:30.\n```\nsome code\n```"; - let out = preprocess_for_tts(input); - assert!(!out.contains("**"), "got: {out}"); - assert!(!out.contains("https://"), "got: {out}"); - assert!(out.contains("eleven thirty"), "got: {out}"); - assert!(out.contains("code block omitted"), "got: {out}"); - } -} +#[allow(unused_imports)] +pub use buzz_voice_pkg::preparation::{ + prepare_tts_chunks, preprocess_for_tts, shape_tts_chunk, split_sentences, +}; diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 63a435cd8e..3592093ae5 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -47,7 +47,9 @@ use std::{ }; use super::pocket::{load_text_to_speech, load_voice_style, SAMPLE_RATE, VOICE_FILE_EXT}; -use super::preprocessing::{preprocess_for_tts, split_sentences}; +use super::preprocessing::{prepare_tts_chunks, shape_tts_chunk}; +#[cfg(test)] +use buzz_voice_pkg::preparation::group_sentences_into_chunks; // ── Constants ───────────────────────────────────────────────────────────────── @@ -69,51 +71,6 @@ const MONITOR_TICK: Duration = Duration::from_millis(10); /// Pocket TTS is a one-step consistency model, not diffusion. Kept for API compat. const SYNTH_STEPS: usize = 1; -/// Fade-out length in samples (8 ms at 24 kHz ≈ 192 samples). -/// -/// Applied only at the *end* of each synthesised sentence to eliminate the -/// click that would otherwise occur when a non-zero waveform terminates -/// abruptly. **No fade-in is applied** — see `apply_fade_out` for the -/// rationale and `examples/pocket_onset_probe.rs` for the measurement that -/// motivated removing the leading fade. -const FADE_OUT_SAMPLES: usize = (SAMPLE_RATE as f64 * 0.008) as usize; - -/// Length of the zero-sample cushion prepended before each synthesized -/// sentence chunk, so the OS audio device / rodio mixer has a fully-quiet -/// ramp-up window before the real onset hits. -/// -/// This used to be applied only before the first sentence of a whole response. -/// That still left later sentence chunks vulnerable to first-syllable clipping -/// when their first phoneme was soft (notably `I'm` / `I've`) and rodio crossed -/// from an explicit silence buffer straight into non-zero speech. 20 ms ≈ 480 -/// samples is enough to cover a CoreAudio buffer turnover without being audible -/// as latency. At sentence boundaries this lead-in is budgeted out of the -/// existing inter-sentence pause, so it does not lengthen multi-sentence gaps. -const SENTENCE_LEAD_IN_SAMPLES: usize = (SAMPLE_RATE as f64 * 0.020) as usize; - -/// Approximate character budget for one synthesis chunk. -/// -/// Upstream pocket-tts groups sentences into chunks of up to -/// `MAX_TOKEN_PER_CHUNK = 50` tokenizer tokens (`default_parameters.py`) — -/// typically multi-sentence chunks — because every `generate()` call is an -/// independent generation with a cold FlowLM start, and each chunk boundary -/// is an exposed prosody seam (kyutai-labs/pocket-tts #151; the Kyutai team -/// names chunk stitching as the reliability lever). Our previous -/// sentence-per-call path created ~2–4× more seams than upstream. -/// -/// We don't ship the SentencePiece tokenizer, so 50 tokens is approximated -/// with a character budget. The bundled 4k-entry vocab averages ~4 chars per -/// token, but usage-weighted English text leans on short common tokens, so -/// the effective ratio is ~2–4 chars/token and 200 chars ≈ 60–100 tokens — -/// modestly above upstream's 50, deliberately: erring large means fewer -/// seams, and even ~100 tokens is far below the model's 500-LM-step (~40 s) -/// ceiling. Do not shrink this budget to chase an exact 50-token match. -const MAX_CHUNK_CHARS: usize = 200; - -/// Silence inserted between sentences by the TTS pipeline (seconds). -/// Injected as a silent buffer between each synthesized sentence chunk. -const INTER_SENTENCE_SILENCE: f32 = 0.1; - // ── Public pipeline handle ──────────────────────────────────────────────────── /// Handle to the running TTS pipeline. @@ -417,11 +374,9 @@ fn tts_worker( // `tts_active` lifecycle: set on the first append while idle, cleared // whenever the player has fully drained — either in the idle timeout // arm or on item receipt before synthesis begins. - let silence_buf_len = (INTER_SENTENCE_SILENCE * SAMPLE_RATE as f32) as usize; // `first_append` = "no audio queued since the player last went idle". - // Flipped by `build_sentence_append_buffer` on the first real append; the - // idle branch below uses it to decide when to drop `tts_active` and to - // arm a fresh lead-in cushion for the next utterance. + // The first successful append clears it; the idle branch below uses it to + // decide when to drop `tts_active`. let mut first_append = true; loop { @@ -485,24 +440,14 @@ fn tts_worker( first_append = true; } - // Preprocess text. - let text = preprocess_for_tts(&raw_text); - if text.is_empty() { - continue; - } - - // Split into sentences, then group into synthesis chunks: the first + // Prepare and group synthesis chunks: the first // sentence stays alone (fast time-to-first-audio), the rest pack // greedily up to MAX_CHUNK_CHARS. Each chunk is one `generate()` // call; playback of chunk N overlaps synthesis of chunk N+1 // (lookahead pipelining). Grouping matches upstream's ~50-token // chunking and halves the exposed prosody seams on multi-sentence // replies — see MAX_CHUNK_CHARS. - let sentences: Vec = split_sentences(&text) - .into_iter() - .filter(|s| !s.trim().is_empty()) - .collect(); - let chunks = group_sentences_into_chunks(&sentences, MAX_CHUNK_CHARS); + let chunks = prepare_tts_chunks(&raw_text); for chunk in &chunks { if handle_cancel_or_shutdown( @@ -523,19 +468,13 @@ fn tts_worker( match engine.synth_chunk(text, "en", &style, SYNTH_STEPS) { Ok(samples) if !samples.is_empty() => { - let mut audio = clamp_to_full_scale(samples); - // Fade-out only — fading-in would attenuate the consonant - // onset (see `apply_fade_out` docstring + the - // 2026-05-18 "first little sound is missing" regression). - apply_fade_out(&mut audio); - // Build one contiguous buffer per synthesized sentence: // lead-in cushion + audio + trailing gap. Keeping this as // a single rodio source preserves the original queue/drain // semantics (one append per sentence) while still giving // every chunk a quiet device warm-up window. - let buf = - build_sentence_append_buffer(&mut first_append, audio, silence_buf_len); + let buf = shape_tts_chunk(samples); + first_append = false; // Check-and-append under `player_ops`, serialized with // the monitor: a barge-in may have arrived during @@ -644,130 +583,6 @@ fn lock_player_ops(ops: &Mutex<()>) -> MutexGuard<'_, ()> { ops.lock().unwrap_or_else(PoisonError::into_inner) } -/// Hard-clamp samples to ±1.0 full scale. -/// -/// No gain is applied: Pocket TTS already emits speech-level audio -/// (peaks 0.4–0.97, RMS ≈ −20 dBFS across varied sentences — measured by -/// `examples/pocket_clip_probe`), matching the kyutai reference pipeline, -/// which applies no output scaling. Two earlier gain stages were both -/// regressions against that baseline: per-sentence peak normalization caused -/// level pumping between sentences, and the fixed 9.3× gain that replaced it -/// was calibrated on a single anomalously-quiet bench utterance (peak 0.076) -/// and clipped 13–34% of samples on real speech ("blown out", 2026-06-12). -/// The clamp alone remains as the safety net against outlier transients. -fn clamp_to_full_scale(samples: Vec) -> Vec { - samples.into_iter().map(|s| s.clamp(-1.0, 1.0)).collect() -} - -/// Apply a short linear fade-out at the *end* of `samples`. -/// -/// Uses `FADE_OUT_SAMPLES` (8 ms) or half the buffer length, whichever is -/// smaller. Eliminates the click that occurs when a non-zero waveform -/// terminates abruptly at a sentence boundary. -/// -/// # Why no fade-in -/// -/// An earlier revision (pre 2026-05) symmetrically faded *in* over the same -/// 8 ms window. That swallowed the leading consonant attack on every -/// sentence — Pocket TTS produces real audio energy inside the first -/// millisecond (RMS ≈ 0.02, peak ≈ 0.03 measured across four prompts in -/// `examples/pocket_onset_probe.rs`), and a linear 0→1 ramp over 192 samples -/// scales those onset samples by ≤50 % for the first ~4 ms. The result was -/// the "first little sound or two is missing" regression heard on -/// 2026-05-18. -/// -/// The first sample of Pocket output measures ≈ 0.0018 (≈ −54 dBFS) — well -/// below the threshold at which a DC-jump would be audible as a click — so -/// no fade-in is needed. The OS audio device gets its quiet ramp-up window -/// from `SENTENCE_LEAD_IN_SAMPLES` instead, inserted as pure silence before -/// each sentence buffer. -fn apply_fade_out(samples: &mut [f32]) { - let len = samples.len(); - let fade = FADE_OUT_SAMPLES.min(len / 2); - for i in 0..fade { - samples[len - 1 - i] *= i as f32 / fade as f32; - } -} - -/// Build the single buffer appended to the rodio `Player` for one synthesised -/// sentence. -/// -/// Every sentence chunk gets a short lead-in pad immediately before its audio. -/// This matters for chunks that start with soft first phonemes (`I'm`, `I've`): -/// the synthesized buffer can begin with speech within the first millisecond, -/// so the playback layer must provide the device/mixer cushion. -/// To keep the audible gap unchanged, the trailing silence after this chunk is -/// shortened by the same amount (`silence_buf_len - SENTENCE_LEAD_IN_SAMPLES`): -/// sentence N contributes 80 ms of post-speech silence and sentence N+1 -/// contributes the remaining 20 ms of pre-speech cushion. -/// -/// The lead-in, audio, and trailing silence are concatenated into one -/// `SamplesBuffer` before appending. This keeps rodio's queue shape at one -/// tracked source per synthesized sentence, avoiding source-boundary/drain -/// regressions from enqueueing the lead-in, audio, and tail as separate sounds. -/// -/// `first_append` is flipped on the first call after the player goes idle. -/// The worker uses it in the idle branch of the main loop to distinguish -/// "never queued anything since last drain" from "drained after speaking", -/// which controls when `tts_active` is released and the lead-in re-armed. -fn build_sentence_append_buffer( - first_append: &mut bool, - audio: Vec, - silence_buf_len: usize, -) -> Vec { - if *first_append { - *first_append = false; - } - - let trailing_silence_len = silence_buf_len.saturating_sub(SENTENCE_LEAD_IN_SAMPLES); - let mut buf = Vec::with_capacity(SENTENCE_LEAD_IN_SAMPLES + audio.len() + trailing_silence_len); - buf.extend(std::iter::repeat_n(0.0_f32, SENTENCE_LEAD_IN_SAMPLES)); - buf.extend(audio); - buf.extend(std::iter::repeat_n(0.0_f32, trailing_silence_len)); - buf -} - -/// Group sentences into synthesis chunks. -/// -/// The first sentence always stands alone — it is what the listener hears -/// first, and synthesizing it by itself keeps time-to-first-audio at the -/// single-sentence cost. Subsequent sentences pack greedily: a sentence -/// joins the current chunk while the combined length stays within -/// `max_chars`; otherwise it starts a new chunk. A single sentence longer -/// than `max_chars` becomes its own chunk unsplit — Pocket TTS handles long -/// single sentences fine (the ceiling is the 500-LM-step default), it's the -/// *seams* we're minimizing. -/// -/// Sentences within a chunk are joined with a single space; sentence-ending -/// punctuation is preserved by `split_sentences`, so the model sees natural -/// multi-sentence prose — the same shape upstream's ~50-token chunker feeds it. -fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec { - let mut chunks: Vec = Vec::new(); - for (i, sentence) in sentences.iter().enumerate() { - let sentence = sentence.trim(); - if sentence.is_empty() { - continue; - } - if i == 0 || chunks.is_empty() { - chunks.push(sentence.to_string()); - continue; - } - // Never merge into the first chunk — it's the latency-critical one. - let can_merge = chunks.len() > 1 - && chunks - .last() - .is_some_and(|c| c.len() + 1 + sentence.len() <= max_chars); - if can_merge { - let last = chunks.last_mut().expect("non-empty checked above"); - last.push(' '); - last.push_str(sentence); - } else { - chunks.push(sentence.to_string()); - } - } - chunks -} - // drain_until_shutdown lives in super (huddle/mod.rs) — shared with stt.rs. use super::drain_until_shutdown; diff --git a/desktop/src-tauri/src/huddle/tts_tests.rs b/desktop/src-tauri/src/huddle/tts_tests.rs index 7887f8bbdb..90d9b147a7 100644 --- a/desktop/src-tauri/src/huddle/tts_tests.rs +++ b/desktop/src-tauri/src/huddle/tts_tests.rs @@ -736,186 +736,6 @@ fn frames_while_tts_inactive_are_not_counted() { ); } -// ── apply_fade_out tests ────────────────────────────────────────────────── - -/// The fade-out half of the old `apply_fades`: last sample is silenced -/// and the ramp is monotonic. Mid-buffer must be untouched. -#[test] -fn apply_fade_out_short_buffer() { - let mut samples = vec![1.0f32; 10]; - apply_fade_out(&mut samples); - assert_eq!(samples[9], 0.0, "last sample should be silenced"); - assert!(samples[5] > 0.5, "mid-buffer should be near-untouched"); -} - -/// REGRESSION (2026-05-18): the *first* samples must NOT be attenuated. -/// An earlier `apply_fades` symmetrically faded in over 8 ms which -/// swallowed the consonant onset of every sentence. -/// Lock in: samples[0..FADE_OUT_SAMPLES] are byte-equal to input. -#[test] -fn apply_fade_out_does_not_touch_leading_samples() { - // Input long enough that fade window doesn't overlap (≫ 2× fade). - let n = FADE_OUT_SAMPLES * 4; - let input: Vec = (0..n).map(|i| 0.5 + (i as f32) * 1e-4).collect(); - let mut samples = input.clone(); - apply_fade_out(&mut samples); - for i in 0..FADE_OUT_SAMPLES { - assert_eq!( - samples[i], input[i], - "leading sample {i} must not be attenuated (was {} → {})", - input[i], samples[i] - ); - } - // And the trailing fade still works. - assert_eq!(samples[n - 1], 0.0); -} - -#[test] -fn apply_fade_out_empty_buffer() { - let mut samples: Vec = vec![]; - apply_fade_out(&mut samples); - assert!(samples.is_empty()); -} - -#[test] -fn apply_fade_out_single_sample() { - // fade = min(FADE_OUT_SAMPLES, len/2) = 0, so nothing changes. - let mut samples = vec![1.0f32]; - apply_fade_out(&mut samples); - assert_eq!(samples[0], 1.0); -} - -/// Sanity-check the per-sentence cushion length: 20 ms at 24 kHz must -/// land at exactly 480 samples. This is a const computation, so the -/// real value of this test is documenting *why* 20 ms was chosen — it -/// covers a typical CoreAudio buffer turnover (256–1024 samples) -/// without being audible as user-facing latency. -#[test] -fn sentence_lead_in_is_sane() { - assert_eq!(SENTENCE_LEAD_IN_SAMPLES, 480, "20 ms × 24 kHz"); -} - -// ── build_sentence_append_buffer tests ─────────────────────────────────── - -/// REGRESSION: every chunk needs an onset cushion; synthesized chunks -/// can start with speech energy within the first millisecond. -#[test] -fn lead_in_pad_is_present_for_every_sentence_chunk() { - const SENTENCE_AUDIO_LEN: usize = 1000; - const SILENCE_BUF_LEN: usize = 2400; // 100 ms at 24 kHz, like production - const N_SENTENCES: usize = 5; - - let mut first = true; - - for _ in 0..N_SENTENCES { - let buf = build_sentence_append_buffer( - &mut first, - vec![0.5_f32; SENTENCE_AUDIO_LEN], - SILENCE_BUF_LEN, - ); - - assert_eq!(buf.len(), SENTENCE_AUDIO_LEN + SILENCE_BUF_LEN); - assert!( - buf[..SENTENCE_LEAD_IN_SAMPLES].iter().all(|&s| s == 0.0), - "lead-in pad must be pure silence" - ); - assert!( - buf[SENTENCE_LEAD_IN_SAMPLES..SENTENCE_LEAD_IN_SAMPLES + SENTENCE_AUDIO_LEN] - .iter() - .all(|&s| s == 0.5), - "sentence audio must immediately follow the lead-in" - ); - assert!( - buf[SENTENCE_LEAD_IN_SAMPLES + SENTENCE_AUDIO_LEN..] - .iter() - .all(|&s| s == 0.0), - "trailing gap must be pure silence" - ); - } - - assert!(!first, "first_append flag must be cleared after first call"); -} - -/// `first_append` still flips on the first call for `tts_active` gating. -#[test] -fn build_sentence_append_buffer_flips_first_append() { - let mut first = true; - let _ = build_sentence_append_buffer(&mut first, vec![0.5; 100], 2400); - assert!(!first, "first call must flip the flag"); - - // Subsequent call: still has a per-sentence lead-in, flag stays false. - let buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], 2400); - assert!(buf[..SENTENCE_LEAD_IN_SAMPLES].iter().all(|&s| s == 0.0)); - assert!(!first); -} - -/// Leading silence is exactly the lead-in; no pre-audio gap is double-counted. -#[test] -fn first_sentence_leading_silence_is_exactly_lead_in() { - let mut first = true; - let buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], 2400); - assert!(buf[..SENTENCE_LEAD_IN_SAMPLES].iter().all(|&s| s == 0.0)); - assert_eq!(buf[SENTENCE_LEAD_IN_SAMPLES], 0.5); -} - -/// Tail silence plus the next lead-in preserves the 100 ms sentence gap. -#[test] -fn sentence_gap_budget_is_preserved() { - let mut first = true; - let silence_buf_len = 2400; - let first_buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], silence_buf_len); - let second_buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], silence_buf_len); - - let first_tail = &first_buf[SENTENCE_LEAD_IN_SAMPLES + 100..]; - let second_lead = &second_buf[..SENTENCE_LEAD_IN_SAMPLES]; - assert_eq!(first_tail.len(), silence_buf_len - SENTENCE_LEAD_IN_SAMPLES); - assert_eq!(second_lead.len(), SENTENCE_LEAD_IN_SAMPLES); - assert_eq!(first_tail.len() + second_lead.len(), silence_buf_len); -} - -/// Regression guard: one contiguous rodio source per synthesized sentence. -#[test] -fn sentence_append_buffer_is_one_contiguous_source() { - let mut first = true; - let buf = build_sentence_append_buffer(&mut first, vec![0.5; 100], 2400); - - assert_eq!(buf.len(), 2400 + 100); - assert!(buf[..SENTENCE_LEAD_IN_SAMPLES].iter().all(|&s| s == 0.0)); - assert!( - buf[SENTENCE_LEAD_IN_SAMPLES..SENTENCE_LEAD_IN_SAMPLES + 100] - .iter() - .all(|&s| s == 0.5) - ); -} - -// ── clamp_to_full_scale tests ───────────────────────────────────────────── - -/// In-range speech audio passes through bit-exact — no gain is applied. -/// (Pocket output is already at speech level; see the fn doc-comment for -/// the history of the two gain-stage regressions this replaced.) -#[test] -fn clamp_to_full_scale_passes_speech_audio_unchanged() { - let input = vec![0.42_f32, -0.97, 0.076, 0.0]; - let out = clamp_to_full_scale(input.clone()); - assert_eq!(out, input); -} - -/// Outlier transients beyond full scale are hard-clamped to ±1.0 rather -/// than wrapping. -#[test] -fn clamp_to_full_scale_clamps_outliers() { - let input = vec![1.5_f32, -2.0, 0.5]; - let out = clamp_to_full_scale(input); - assert_eq!(out, vec![1.0, -1.0, 0.5]); -} - -/// Empty input round-trips to empty output. -#[test] -fn clamp_to_full_scale_empty_buffer() { - let out = clamp_to_full_scale(Vec::new()); - assert!(out.is_empty()); -} - // ── group_sentences_into_chunks tests ───────────────────────────────────── fn s(v: &[&str]) -> Vec { diff --git a/mobile/android/app/build.gradle.kts b/mobile/android/app/build.gradle.kts index 501a31c46f..3b0b966ece 100644 --- a/mobile/android/app/build.gradle.kts +++ b/mobile/android/app/build.gradle.kts @@ -92,8 +92,15 @@ android { versionName = flutter.versionName testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" resValue("string", "app_name", "Buzz") + ndk { + // Native Pocket voice artifacts are built for device arm64 and + // emulator x86_64. Do not publish an installable ABI without them. + abiFilters += listOf("arm64-v8a", "x86_64") + } } + sourceSets["main"].jniLibs.srcDir("../../.generated/voice/android/jniLibs") + signingConfigs { if (hasUploadSigning) { create("upload") { @@ -124,6 +131,20 @@ android { } } +val buildBuzzVoiceNative by tasks.registering(Exec::class) { + workingDir(rootProject.projectDir.parentFile.parentFile) + environment("ANDROID_NDK_HOME", android.ndkDirectory.absolutePath) + commandLine( + "/bin/bash", + "-c", + ". ./bin/activate-hermit && ./scripts/mobile-voice-native.sh build-android", + ) +} + +tasks.named("preBuild").configure { + dependsOn(buildBuzzVoiceNative) +} + dependencies { testImplementation(kotlin("test")) diff --git a/mobile/android/app/src/main/AndroidManifest.xml b/mobile/android/app/src/main/AndroidManifest.xml index 17a16742af..b63e4d40ba 100644 --- a/mobile/android/app/src/main/AndroidManifest.xml +++ b/mobile/android/app/src/main/AndroidManifest.xml @@ -4,6 +4,8 @@ + if (shouldStopPocketVoiceForAudioFocusChange(change)) { + stopAndNotify("interrupted") + } + } + private val noisyReceiver = + object : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + stopAndNotify("routeLost") + } + } + + init { + channel.setMethodCallHandler(::handle) + applicationContext.registerReceiver( + noisyReceiver, + IntentFilter(AudioManager.ACTION_AUDIO_BECOMING_NOISY), + ) + } + + fun background() { + mainHandler.post { + stop(notify = null) + channel.invokeMethod("backgrounded", null) + } + } + + fun dispose() { + stop(notify = null) + channel.setMethodCallHandler(null) + runCatching { applicationContext.unregisterReceiver(noisyReceiver) } + } + + private fun handle( + call: MethodCall, + result: MethodChannel.Result, + ) { + when (call.method) { + "play" -> { + val pcm = call.argument("pcm") + val sampleRate = call.argument("sampleRate") + if (pcm == null || sampleRate == null) { + result.error("invalid_arguments", "Expected PCM and sample rate.", null) + return + } + runCatching { play(pcm, sampleRate) } + .onSuccess { result.success(null) } + .onFailure { + result.error("playback_failed", it.message, null) + } + } + "stop" -> { + stop(notify = null) + result.success(null) + } + "availableCapacity" -> { + val path = call.arguments as? String + if (path == null) { + result.error("invalid_arguments", "Expected a storage path.", null) + } else { + runCatching { StatFs(path).availableBytes } + .onSuccess(result::success) + .onFailure { result.error("storage_failed", it.message, null) } + } + } + "excludeFromBackup" -> result.success(null) + else -> result.notImplemented() + } + } + + private fun play( + pcm: ByteArray, + sampleRate: Int, + ) { + stop(notify = null) + check(requestFocus()) { "Audio focus was denied." } + val nextTrack = + try { + val attributes = + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_ASSISTANCE_ACCESSIBILITY) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + val format = + AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(sampleRate) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .build() + val minimum = AudioTrack.getMinBufferSize( + sampleRate, + AudioFormat.CHANNEL_OUT_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ) + AudioTrack.Builder() + .setAudioAttributes(attributes) + .setAudioFormat(format) + .setTransferMode(AudioTrack.MODE_STREAM) + .setBufferSizeInBytes(max(minimum, 32 * 1024)) + .build() + .also { + if (it.state != AudioTrack.STATE_INITIALIZED) { + it.release() + error("Unable to initialize voice audio.") + } + } + } catch (error: Exception) { + abandonFocus() + throw error + } + track = nextTrack + val currentGeneration = generation.incrementAndGet() + try { + nextTrack.play() + } catch (error: Exception) { + track = null + nextTrack.release() + abandonFocus() + throw error + } + thread(name = "buzz-pocket-playback") { + var offset = 0 + var complete = false + var started = false + fun reportStartedIfPlaybackAdvanced() { + if (started || generation.get() != currentGeneration) return + val playedFrames = + runCatching { + nextTrack.playbackHeadPosition.toLong() and 0xFFFF_FFFFL + }.getOrNull() ?: return + if (playedFrames == 0L) return + started = true + mainHandler.post { + if (generation.get() == currentGeneration && track === nextTrack) { + channel.invokeMethod("started", null) + } + } + } + try { + while (offset < pcm.size && generation.get() == currentGeneration) { + val written = nextTrack.write( + pcm, + offset, + pcm.size - offset, + AudioTrack.WRITE_BLOCKING, + ) + if (written <= 0) break + offset += written + reportStartedIfPlaybackAdvanced() + } + complete = offset == pcm.size + if (complete) { + val targetFrames = pcm.size.toLong() / 2 + val deadlineNanos = + System.nanoTime() + + (targetFrames * 1_000_000_000L / sampleRate) + + 2_000_000_000L + var drained = false + while ( + generation.get() == currentGeneration && + System.nanoTime() < deadlineNanos + ) { + val playedFrames = + runCatching { + nextTrack.playbackHeadPosition.toLong() and 0xFFFF_FFFFL + }.getOrNull() ?: break + reportStartedIfPlaybackAdvanced() + if (playedFrames >= targetFrames) { + drained = true + break + } + Thread.sleep(5) + } + complete = drained + } + } catch (_: Exception) { + complete = false + } + mainHandler.post { + if (generation.get() == currentGeneration && track === nextTrack) { + releaseTrack(nextTrack) + track = null + abandonFocus() + channel.invokeMethod( + if (complete) "completed" else "error", + null, + ) + } + } + } + } + + private fun stopAndNotify(event: String) { + mainHandler.post { stop(notify = event) } + } + + private fun stop(notify: String?) { + if (track == null && !hasFocus) return + generation.incrementAndGet() + track?.let(::releaseTrack) + track = null + abandonFocus() + if (notify != null) channel.invokeMethod(notify, null) + } + + private fun requestFocus(): Boolean { + val result = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val request = + AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK) + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_ASSISTANCE_ACCESSIBILITY) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build(), + ) + .setOnAudioFocusChangeListener(focusListener, mainHandler) + .build() + focusRequest = request + audioManager.requestAudioFocus(request) + } else { + @Suppress("DEPRECATION") + audioManager.requestAudioFocus( + focusListener, + AudioManager.STREAM_MUSIC, + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK, + ) + } + hasFocus = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + return hasFocus + } + + private fun abandonFocus() { + if (!hasFocus) return + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + focusRequest?.let(audioManager::abandonAudioFocusRequest) + } else { + @Suppress("DEPRECATION") + audioManager.abandonAudioFocus(focusListener) + } + focusRequest = null + hasFocus = false + } + + private fun releaseTrack(audioTrack: AudioTrack) { + runCatching { audioTrack.pause() } + runCatching { audioTrack.flush() } + runCatching { audioTrack.stop() } + runCatching { audioTrack.release() } + } + + private companion object { + const val CHANNEL = "buzz/voice_audio" + } +} diff --git a/mobile/android/app/src/main/res/xml/backup_rules.xml b/mobile/android/app/src/main/res/xml/backup_rules.xml new file mode 100644 index 0000000000..188f46cf7a --- /dev/null +++ b/mobile/android/app/src/main/res/xml/backup_rules.xml @@ -0,0 +1,4 @@ + + + + diff --git a/mobile/android/app/src/main/res/xml/data_extraction_rules.xml b/mobile/android/app/src/main/res/xml/data_extraction_rules.xml new file mode 100644 index 0000000000..263cda4139 --- /dev/null +++ b/mobile/android/app/src/main/res/xml/data_extraction_rules.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/mobile/android/app/src/test/kotlin/xyz/block/buzz/mobile/VoiceAudioOutputTest.kt b/mobile/android/app/src/test/kotlin/xyz/block/buzz/mobile/VoiceAudioOutputTest.kt new file mode 100644 index 0000000000..f35963e01b --- /dev/null +++ b/mobile/android/app/src/test/kotlin/xyz/block/buzz/mobile/VoiceAudioOutputTest.kt @@ -0,0 +1,28 @@ +package xyz.block.buzz.mobile + +import android.media.AudioManager +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class VoiceAudioOutputTest { + @Test + fun `all focus losses stop Pocket voice`() { + assertTrue(shouldStopPocketVoiceForAudioFocusChange(AudioManager.AUDIOFOCUS_LOSS)) + assertTrue( + shouldStopPocketVoiceForAudioFocusChange( + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT, + ), + ) + assertTrue( + shouldStopPocketVoiceForAudioFocusChange( + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK, + ), + ) + } + + @Test + fun `focus gain does not interrupt Pocket voice`() { + assertFalse(shouldStopPocketVoiceForAudioFocusChange(AudioManager.AUDIOFOCUS_GAIN)) + } +} diff --git a/mobile/integration_test/pocket_voice_integration_test.dart b/mobile/integration_test/pocket_voice_integration_test.dart new file mode 100644 index 0000000000..c71be32bf7 --- /dev/null +++ b/mobile/integration_test/pocket_voice_integration_test.dart @@ -0,0 +1,164 @@ +import 'dart:io'; + +import 'package:buzz/shared/voice/pocket_model_downloader.dart'; +import 'package:buzz/shared/voice/pocket_model_manifest.dart'; +import 'package:buzz/shared/voice/pocket_voice_worker.dart'; +import 'package:buzz/shared/voice/voice_audio_output.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + testWidgets( + 'Pocket worker synthesizes valid PCM and cancels promptly', + (tester) async { + const configuredModelPath = String.fromEnvironment( + 'BUZZ_POCKET_MODEL_DIR', + ); + const configuredPrecision = String.fromEnvironment( + 'BUZZ_POCKET_PRECISION', + defaultValue: 'fp32', + ); + final variant = PocketModelVariant.fromStored(configuredPrecision); + const downloadModel = bool.fromEnvironment('BUZZ_POCKET_DOWNLOAD'); + var modelPath = configuredModelPath; + Duration? downloadTime; + if (modelPath.isEmpty && downloadModel) { + final clock = Stopwatch()..start(); + final downloader = PocketModelDownloader(variant: variant); + final directory = await downloader.install((downloaded, total, file) { + if (downloaded == total) { + debugPrint('Pocket model download complete: $file ($total bytes)'); + } + }); + clock.stop(); + downloadTime = clock.elapsed; + modelPath = directory.path; + expect(await downloader.verify(directory, hashContents: true), isTrue); + } + if (modelPath.isEmpty) { + fail( + 'Pass --dart-define=BUZZ_POCKET_MODEL_DIR= or ' + '--dart-define=BUZZ_POCKET_DOWNLOAD=true.', + ); + } + + final worker = PocketVoiceWorker(); + final loadClock = Stopwatch()..start(); + await worker.start(modelPath, precision: variant.nativePrecision); + loadClock.stop(); + addTearDown(worker.dispose); + + final firstClock = Stopwatch()..start(); + final firstPlaybackClock = Stopwatch()..start(); + worker.synthesize(1, 'Pocket voice is running on this mobile device.'); + final first = await worker.responses + .where((response) => response is PocketWorkerAudio) + .cast() + .first + .timeout(const Duration(seconds: 30)); + firstClock.stop(); + final bytes = first.data.materialize().asUint8List(); + _expectValidPcm(bytes, first.sampleRate); + final audioSeconds = bytes.length / 2 / first.sampleRate; + final rtf = + first.synthesisTime.inMicroseconds / + Duration.microsecondsPerSecond / + audioSeconds; + final output = PlatformVoiceAudioOutput(); + final playbackStarted = output.events.firstWhere( + (event) => event == VoiceAudioEvent.started, + ); + final playbackCompleted = output.events.firstWhere( + (event) => event == VoiceAudioEvent.completed, + ); + final playbackClock = Stopwatch()..start(); + await output.play(bytes, first.sampleRate); + await playbackStarted.timeout(const Duration(seconds: 30)); + firstPlaybackClock.stop(); + await playbackCompleted.timeout(const Duration(seconds: 30)); + playbackClock.stop(); + expect( + playbackClock.elapsedMilliseconds, + greaterThanOrEqualTo((audioSeconds * 800).floor()), + ); + + final warmClock = Stopwatch()..start(); + worker.synthesize(2, 'The engine stays warm for the next response.'); + final warm = await worker.responses + .where((response) => response is PocketWorkerAudio) + .cast() + .where((audio) => audio.generation == 2) + .first + .timeout(const Duration(seconds: 30)); + warmClock.stop(); + _expectValidPcm(warm.data.materialize().asUint8List(), warm.sampleRate); + + final queuedCancelClock = Stopwatch()..start(); + worker.synthesize( + 3, + List.filled( + 20, + 'This queued request must remain cancelled before synthesis begins.', + ).join(' '), + ); + worker.cancel(); + final queuedCancelled = await worker.responses + .where((response) => response is PocketWorkerFailure) + .cast() + .where((failure) => failure.generation == 3) + .first + .timeout(const Duration(seconds: 10)); + queuedCancelClock.stop(); + expect(queuedCancelled.kind, PocketWorkerFailureKind.cancelled); + + final cancelClock = Stopwatch()..start(); + worker.synthesize( + 4, + List.filled( + 20, + 'This deliberately long sentence gives cancellation time to interrupt.', + ).join(' '), + ); + await Future.delayed(const Duration(milliseconds: 50)); + worker.cancel(); + final cancelled = await worker.responses + .where((response) => response is PocketWorkerFailure) + .cast() + .where((failure) => failure.generation == 4) + .first + .timeout(const Duration(seconds: 10)); + cancelClock.stop(); + expect(cancelled.kind, PocketWorkerFailureKind.cancelled); + + // Emitted in a stable shape so simulator/device runs are easy to compare. + debugPrint( + 'BUZZ_POCKET_METRICS ' + 'precision=${variant.precision} ' + 'load_ms=${loadClock.elapsedMilliseconds} ' + 'first_pcm_ms=${firstClock.elapsedMilliseconds} ' + 'ttfap_ms=${firstPlaybackClock.elapsedMilliseconds} ' + 'synthesis_ms=${first.synthesisTime.inMilliseconds} ' + 'audio_s=${audioSeconds.toStringAsFixed(3)} ' + 'rtf=${rtf.toStringAsFixed(3)} ' + 'playback_ms=${playbackClock.elapsedMilliseconds} ' + 'warm_pcm_ms=${warmClock.elapsedMilliseconds} ' + 'queued_cancel_ms=${queuedCancelClock.elapsedMilliseconds} ' + 'cancel_ms=${cancelClock.elapsedMilliseconds} ' + 'download_ms=${downloadTime?.inMilliseconds ?? 0} ' + 'rss_bytes=${ProcessInfo.currentRss} ' + 'peak_rss_bytes=${ProcessInfo.maxRss}', + ); + }, + timeout: const Timeout(Duration(minutes: 20)), + ); +} + +void _expectValidPcm(Uint8List bytes, int sampleRate) { + expect(sampleRate, 24000); + expect(bytes.length, greaterThan(4800)); + expect(bytes.length % 2, 0); + expect(bytes.any((byte) => byte != 0), isTrue); +} diff --git a/mobile/ios/Flutter/Debug.xcconfig b/mobile/ios/Flutter/Debug.xcconfig index 1f35905051..7ea6b87f99 100644 --- a/mobile/ios/Flutter/Debug.xcconfig +++ b/mobile/ios/Flutter/Debug.xcconfig @@ -19,3 +19,4 @@ APP_DISPLAY_NAME = Buzz // device signing beats the worktree default while unset variables still // fall through to the worktree values. #include? "AppOverrides.xcconfig" +#include "Voice.xcconfig" diff --git a/mobile/ios/Flutter/Release.xcconfig b/mobile/ios/Flutter/Release.xcconfig index d287c5fb43..e16556116c 100644 --- a/mobile/ios/Flutter/Release.xcconfig +++ b/mobile/ios/Flutter/Release.xcconfig @@ -9,3 +9,4 @@ APP_DISPLAY_NAME = Buzz CODE_SIGN_STYLE = Automatic CODE_SIGN_IDENTITY = iPhone Developer #include? "AppOverrides.xcconfig" +#include "Voice.xcconfig" diff --git a/mobile/ios/Flutter/Voice.xcconfig b/mobile/ios/Flutter/Voice.xcconfig new file mode 100644 index 0000000000..ed4fd00bbf --- /dev/null +++ b/mobile/ios/Flutter/Voice.xcconfig @@ -0,0 +1,2 @@ +BUZZ_VOICE_LIB_DIR = $(SRCROOT)/../.generated/voice/ios/$(PLATFORM_NAME)-$(CURRENT_ARCH) +OTHER_LDFLAGS = $(inherited) -Wl,-force_load,$(BUZZ_VOICE_LIB_DIR)/libbuzz_voice_mobile.a -lc++ -framework Accelerate -framework CoreML -framework Foundation -framework Metal -framework QuartzCore -framework Security diff --git a/mobile/ios/Podfile.lock b/mobile/ios/Podfile.lock index ba8c828d25..8ef612f5ff 100644 --- a/mobile/ios/Podfile.lock +++ b/mobile/ios/Podfile.lock @@ -15,6 +15,8 @@ PODS: - FlutterMacOS - image_picker_ios (0.0.1): - Flutter + - integration_test (0.0.1): + - Flutter - mobile_scanner (7.0.0): - Flutter - FlutterMacOS @@ -40,6 +42,7 @@ DEPENDENCIES: - Flutter (from `Flutter`) - flutter_secure_storage_darwin (from `.symlinks/plugins/flutter_secure_storage_darwin/darwin`) - image_picker_ios (from `.symlinks/plugins/image_picker_ios/ios`) + - integration_test (from `.symlinks/plugins/integration_test/ios`) - mobile_scanner (from `.symlinks/plugins/mobile_scanner/darwin`) - open_filex (from `.symlinks/plugins/open_filex/ios`) - package_info_plus (from `.symlinks/plugins/package_info_plus/ios`) @@ -64,6 +67,8 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/flutter_secure_storage_darwin/darwin" image_picker_ios: :path: ".symlinks/plugins/image_picker_ios/ios" + integration_test: + :path: ".symlinks/plugins/integration_test/ios" mobile_scanner: :path: ".symlinks/plugins/mobile_scanner/darwin" open_filex: @@ -86,6 +91,7 @@ SPEC CHECKSUMS: Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467 flutter_secure_storage_darwin: acdb3f316ed05a3e68f856e0353b133eec373a23 image_picker_ios: e0ece4aa2a75771a7de3fa735d26d90817041326 + integration_test: 4a889634ef21a45d28d50d622cf412dc6d9f586e mobile_scanner: 9157936403f5a0644ca3779a38ff8404c5434a93 open_filex: 432f3cd11432da3e39f47fcc0df2b1603854eff1 package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499 diff --git a/mobile/ios/Runner.xcodeproj/project.pbxproj b/mobile/ios/Runner.xcodeproj/project.pbxproj index 46c61ce2fc..8a92f373dc 100644 --- a/mobile/ios/Runner.xcodeproj/project.pbxproj +++ b/mobile/ios/Runner.xcodeproj/project.pbxproj @@ -17,6 +17,7 @@ 42C129326CE4E1B8E617B9CD /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = CD6B899582D0416ADBD8A68F /* Pods_RunnerTests.framework */; }; 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 74858FAE1ED2DC5600515810 /* AppDelegate.swift */; }; 7884E8682EC3CC0700C636F2 /* SceneDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7884E8672EC3CC0400C636F2 /* SceneDelegate.swift */; }; + A70D8C8D31879E5200B02246 /* VoiceAudioOutput.swift in Sources */ = {isa = PBXBuildFile; fileRef = A70D8C8C31879E5200B02246 /* VoiceAudioOutput.swift */; }; 97C146FC1CF9000F007C117D /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FA1CF9000F007C117D /* Main.storyboard */; }; 97C146FE1CF9000F007C117D /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FD1CF9000F007C117D /* Assets.xcassets */; }; 97C147011CF9000F007C117D /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */; }; @@ -72,6 +73,7 @@ 97C146FD1CF9000F007C117D /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 97C147001CF9000F007C117D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 97C147021CF9000F007C117D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + A70D8C8C31879E5200B02246 /* VoiceAudioOutput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceAudioOutput.swift; sourceTree = ""; }; CD6B899582D0416ADBD8A68F /* Pods_RunnerTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_RunnerTests.framework; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ @@ -171,6 +173,7 @@ 74858FAE1ED2DC5600515810 /* AppDelegate.swift */, 331C809A294A618700263BE5 /* MediaSanitizer.swift */, 7884E8672EC3CC0400C636F2 /* SceneDelegate.swift */, + A70D8C8C31879E5200B02246 /* VoiceAudioOutput.swift */, 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */, ); path = Runner; @@ -213,6 +216,7 @@ buildPhases = ( 1D327B920BBA7EF4545E2A92 /* [CP] Check Pods Manifest.lock */, 9740EEB61CF901F6004384FC /* Run Script */, + A70D8C8E31879E5200B02246 /* Build Pocket Voice */, 97C146EA1CF9000F007C117D /* Sources */, 97C146EB1CF9000F007C117D /* Frameworks */, 97C146EC1CF9000F007C117D /* Resources */, @@ -367,6 +371,22 @@ shellPath = /bin/sh; shellScript = "/bin/sh \"$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh\" build"; }; + A70D8C8E31879E5200B02246 /* Build Pocket Voice */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + ); + name = "Build Pocket Voice"; + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/bash; + shellScript = "cd \"$SRCROOT/../..\"\n. ./bin/activate-hermit\n./scripts/mobile-voice-native.sh build-ios-current\n"; + showEnvVarsInLog = 0; + }; E0B5862D106D142B580309AF /* [CP] Embed Pods Frameworks */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; @@ -403,6 +423,7 @@ 331C809B294A63AB00263BE5 /* MediaSanitizer.swift in Sources */, 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */, 7884E8682EC3CC0700C636F2 /* SceneDelegate.swift in Sources */, + A70D8C8D31879E5200B02246 /* VoiceAudioOutput.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/mobile/ios/Runner/AppDelegate.swift b/mobile/ios/Runner/AppDelegate.swift index 5edb4b6aaf..47e3b06c5f 100644 --- a/mobile/ios/Runner/AppDelegate.swift +++ b/mobile/ios/Runner/AppDelegate.swift @@ -7,6 +7,7 @@ import UserNotifications @objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate { private var mediaUploadChannel: FlutterMethodChannel? private var qrScannerChannel: FlutterMethodChannel? + private var voiceAudioOutput: VoiceAudioOutput? override func application( _ application: UIApplication, @@ -32,6 +33,9 @@ import UserNotifications qrScannerChannel?.setMethodCallHandler { call, result in Self.handleQrScannerMethodCall(call, result: result) } + voiceAudioOutput = VoiceAudioOutput( + messenger: engineBridge.applicationRegistrar.messenger() + ) } private static func handleQrScannerMethodCall( diff --git a/mobile/ios/Runner/Runner-Bridging-Header.h b/mobile/ios/Runner/Runner-Bridging-Header.h index 308a2a560b..b421e7bbb2 100644 --- a/mobile/ios/Runner/Runner-Bridging-Header.h +++ b/mobile/ios/Runner/Runner-Bridging-Header.h @@ -1 +1,2 @@ #import "GeneratedPluginRegistrant.h" +#import "../../../crates/buzz-voice-mobile/include/buzz_voice_mobile.h" diff --git a/mobile/ios/Runner/VoiceAudioOutput.swift b/mobile/ios/Runner/VoiceAudioOutput.swift new file mode 100644 index 0000000000..3340c0d4dd --- /dev/null +++ b/mobile/ios/Runner/VoiceAudioOutput.swift @@ -0,0 +1,208 @@ +import AVFoundation +import Flutter +import UIKit + +final class VoiceAudioOutput: NSObject, AVAudioPlayerDelegate { + private let channel: FlutterMethodChannel + private var player: AVAudioPlayer? + private var observers: [NSObjectProtocol] = [] + + init(messenger: FlutterBinaryMessenger) { + channel = FlutterMethodChannel( + name: "buzz/voice_audio", + binaryMessenger: messenger + ) + super.init() + channel.setMethodCallHandler { [weak self] call, result in + self?.handle(call, result: result) + } + observeAudioLifecycle() + } + + deinit { + for observer in observers { + NotificationCenter.default.removeObserver(observer) + } + } + + private func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "play": + guard + let arguments = call.arguments as? [String: Any], + let typedData = arguments["pcm"] as? FlutterStandardTypedData, + let sampleRate = arguments["sampleRate"] as? Int + else { + result(FlutterError(code: "invalid_arguments", message: "Expected PCM and sample rate.", details: nil)) + return + } + do { + try play(pcm: typedData.data, sampleRate: sampleRate) + result(nil) + } catch { + result(FlutterError(code: "playback_failed", message: error.localizedDescription, details: nil)) + } + case "stop": + stop(deactivate: true) + result(nil) + case "availableCapacity": + guard let path = call.arguments as? String else { + result(FlutterError(code: "invalid_arguments", message: "Expected a storage path.", details: nil)) + return + } + do { + let values = try URL(fileURLWithPath: path).resourceValues( + forKeys: [.volumeAvailableCapacityForImportantUsageKey] + ) + result(values.volumeAvailableCapacityForImportantUsage ?? 0) + } catch { + result(FlutterError(code: "storage_failed", message: error.localizedDescription, details: nil)) + } + case "excludeFromBackup": + guard let path = call.arguments as? String else { + result(FlutterError(code: "invalid_arguments", message: "Expected a storage path.", details: nil)) + return + } + do { + var url = URL(fileURLWithPath: path) + var values = URLResourceValues() + values.isExcludedFromBackup = true + try url.setResourceValues(values) + result(nil) + } catch { + result(FlutterError(code: "storage_failed", message: error.localizedDescription, details: nil)) + } + default: + result(FlutterMethodNotImplemented) + } + } + + private func play(pcm: Data, sampleRate: Int) throws { + stop(deactivate: false) + let session = AVAudioSession.sharedInstance() + try Self.configureAudioSession(session) + try session.setActive(true) + do { + let nextPlayer = try AVAudioPlayer(data: Self.waveData(pcm: pcm, sampleRate: sampleRate)) + nextPlayer.delegate = self + nextPlayer.prepareToPlay() + guard nextPlayer.play() else { + throw NSError( + domain: "BuzzVoice", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Unable to start voice playback."] + ) + } + player = nextPlayer + channel.invokeMethod("started", arguments: nil) + } catch { + try? session.setActive(false, options: [.notifyOthersOnDeactivation]) + throw error + } + } + + static func configureAudioSession(_ session: AVAudioSession) throws { + try session.setCategory(.playback, mode: .spokenAudio, options: [.duckOthers]) + } + + private func stop(deactivate: Bool) { + player?.stop() + player = nil + if deactivate { + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + } + } + + func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { + guard self.player === player else { return } + self.player = nil + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + channel.invokeMethod(flag ? "completed" : "error", arguments: nil) + } + + func audioPlayerDecodeErrorDidOccur(_ player: AVAudioPlayer, error: Error?) { + guard self.player === player else { return } + self.player = nil + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + channel.invokeMethod("error", arguments: error?.localizedDescription) + } + + private func observeAudioLifecycle() { + let center = NotificationCenter.default + observers.append( + center.addObserver( + forName: AVAudioSession.interruptionNotification, + object: nil, + queue: .main + ) { [weak self] notification in + guard + let rawType = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt, + AVAudioSession.InterruptionType(rawValue: rawType) == .began + else { return } + self?.interrupt(with: "interrupted") + } + ) + observers.append( + center.addObserver( + forName: AVAudioSession.routeChangeNotification, + object: nil, + queue: .main + ) { [weak self] notification in + guard + let rawReason = notification.userInfo?[AVAudioSessionRouteChangeReasonKey] as? UInt, + AVAudioSession.RouteChangeReason(rawValue: rawReason) == .oldDeviceUnavailable + else { return } + self?.interrupt(with: "routeLost") + } + ) + observers.append( + center.addObserver( + forName: UIApplication.didEnterBackgroundNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.interrupt(with: "backgrounded", requirePlayback: false) + } + ) + } + + private func interrupt(with event: String, requirePlayback: Bool = true) { + if requirePlayback && player == nil { return } + stop(deactivate: true) + channel.invokeMethod(event, arguments: nil) + } + + private static func waveData(pcm: Data, sampleRate: Int) -> Data { + var output = Data() + output.append(contentsOf: "RIFF".utf8) + output.appendLittleEndian(UInt32(pcm.count + 36)) + output.append(contentsOf: "WAVEfmt ".utf8) + output.appendLittleEndian(UInt32(16)) + output.appendLittleEndian(UInt16(1)) + output.appendLittleEndian(UInt16(1)) + output.appendLittleEndian(UInt32(sampleRate)) + output.appendLittleEndian(UInt32(sampleRate * 2)) + output.appendLittleEndian(UInt16(2)) + output.appendLittleEndian(UInt16(16)) + output.append(contentsOf: "data".utf8) + output.appendLittleEndian(UInt32(pcm.count)) + output.append(pcm) + return output + } +} + +private extension Data { + mutating func appendLittleEndian(_ value: T) { + var littleEndian = value.littleEndian + Swift.withUnsafeBytes(of: &littleEndian) { append(contentsOf: $0) } + } +} diff --git a/mobile/ios/RunnerTests/RunnerTests.swift b/mobile/ios/RunnerTests/RunnerTests.swift index a174c78684..d8fd610bf6 100644 --- a/mobile/ios/RunnerTests/RunnerTests.swift +++ b/mobile/ios/RunnerTests/RunnerTests.swift @@ -1,4 +1,6 @@ import Flutter +import Darwin +import AVFoundation import UIKit import XCTest @@ -6,6 +8,67 @@ import XCTest class RunnerTests: XCTestCase { + func testPocketVoiceUsesSpokenPlaybackAudioSession() throws { + let session = AVAudioSession.sharedInstance() + try VoiceAudioOutput.configureAudioSession(session) + XCTAssertEqual(session.category, .playback) + XCTAssertEqual(session.mode, .spokenAudio) + XCTAssertTrue(session.categoryOptions.contains(.duckOthers)) + } + + func testPocketVoiceProducesValidPcmWhenModelIsAvailable() throws { + let modelPath = + ProcessInfo.processInfo.environment["BUZZ_POCKET_MODEL_DIR"] + ?? UserDefaults.standard.string(forKey: "BuzzPocketModelTestPath") + guard let modelPath, FileManager.default.fileExists(atPath: modelPath) else { + throw XCTSkip("Set BUZZ_POCKET_MODEL_DIR to run the Pocket TTS integration test.") + } + + let loadStart = ContinuousClock.now + let engineResult = modelPath.withCString { + buzz_voice_engine_create($0, UInt8(BUZZ_VOICE_PRECISION_FP32)) + } + let loadTime = loadStart.duration(to: .now) + if let error = engineResult.error { + defer { buzz_voice_string_free(error) } + XCTFail(String(cString: error)) + return + } + let engine = try XCTUnwrap(engineResult.engine) + defer { buzz_voice_engine_destroy(engine) } + + let prompt = "Pocket voice is running on this iOS simulator." + let firstStart = ContinuousClock.now + let first = prompt.withCString { buzz_voice_engine_synthesize(engine, $0) } + let firstTime = firstStart.duration(to: .now) + let firstData = try validatedPcmData(first) + buzz_voice_pcm_free(first) + + let warmStart = ContinuousClock.now + let warm = prompt.withCString { buzz_voice_engine_synthesize(engine, $0) } + let warmTime = warmStart.duration(to: .now) + let warmData = try validatedPcmData(warm) + buzz_voice_pcm_free(warm) + + var usage = rusage() + getrusage(RUSAGE_SELF, &usage) + let audioSeconds = Double(firstData.count) / 2 / 24_000 + let firstSeconds = durationSeconds(firstTime) + let metrics = + "BUZZ_POCKET_METRICS " + + "load_s=\(durationSeconds(loadTime)) " + + "first_pcm_s=\(firstSeconds) " + + "audio_s=\(audioSeconds) " + + "rtf=\(firstSeconds / audioSeconds) " + + "warm_pcm_s=\(durationSeconds(warmTime)) " + + "peak_rss_bytes=\(usage.ru_maxrss) " + + "repeat_pcm_equal=\(firstData == warmData)" + let attachment = XCTAttachment(string: metrics) + attachment.name = metrics + attachment.lifetime = .keepAlways + add(attachment) + } + func testDynamicIslandQrScannerRecognizesTallSafeAreas() { for safeAreaTopInset in [51, 59, 62] { XCTAssertTrue( @@ -226,6 +289,27 @@ class RunnerTests: XCTestCase { Bundle(for: RunnerTests.self).url(forResource: name, withExtension: fileExtension)) return try Data(contentsOf: url) } + + private func validatedPcmData(_ pcm: BuzzVoicePcm) throws -> Data { + if let error = pcm.error { + defer { buzz_voice_string_free(error) } + XCTFail(String(cString: error)) + return Data() + } + XCTAssertEqual(pcm.sample_rate, 24_000) + XCTAssertGreaterThan(pcm.len, 2_400) + let samples = try XCTUnwrap(pcm.samples) + let buffer = UnsafeBufferPointer(start: samples, count: pcm.len) + XCTAssertTrue(buffer.contains { $0 != 0 }) + XCTAssertGreaterThan(buffer.map { abs(Int32($0)) }.max() ?? 0, 100) + return Data(bytes: samples, count: pcm.len * MemoryLayout.size) + } + + private func durationSeconds(_ duration: Duration) -> Double { + let components = duration.components + return Double(components.seconds) + + Double(components.attoseconds) / 1_000_000_000_000_000_000 + } } private enum RelayImagePolicyError: Error { diff --git a/mobile/lib/features/channels/channel_detail_page.dart b/mobile/lib/features/channels/channel_detail_page.dart index f94226ac80..36a8631df1 100644 --- a/mobile/lib/features/channels/channel_detail_page.dart +++ b/mobile/lib/features/channels/channel_detail_page.dart @@ -13,6 +13,7 @@ import '../../shared/widgets/avatar_image.dart'; import '../../shared/widgets/frosted_app_bar.dart'; import '../../shared/widgets/frosted_scaffold.dart'; import '../../shared/widgets/skeleton.dart'; +import '../../shared/voice/pocket_voice_controller.dart'; import '../profile/presence_cache_provider.dart'; import '../profile/profile_provider.dart'; import '../profile/user_cache_provider.dart'; @@ -35,6 +36,8 @@ import 'members_sheet.dart'; import 'message_actions.dart'; import 'message_content.dart'; import 'mentions/mention_candidates_provider.dart'; +import 'pocket_voice_button.dart'; +import 'pocket_voice_conversation.dart'; import 'read_state/deferred_read_state_update.dart'; import 'read_state/read_state_provider.dart'; import 'read_state/read_state_time.dart'; @@ -123,12 +126,32 @@ class ChannelDetailPage extends HookConsumerWidget { final detailsAsync = ref.watch(channelDetailsProvider(channel.id)); final channelsAsync = ref.watch(channelsProvider); final messagesState = ref.watch(channelMessagesProvider(channel.id)); + final membersState = ref.watch(channelMembersProvider(channel.id)); + final voiceConversation = useMemoized(PocketVoiceConversation.new); final sessionStatus = ref.watch(relaySessionProvider).status; final readState = ref.watch(readStateProvider); final currentPubkey = ref .watch(profileProvider) .whenData((value) => value?.pubkey) .value; + final voiceEventKey = + messagesState.value?.map((event) => event.id).join(',') ?? ''; + useEffect(() { + if (channel.isForum) return null; + final events = messagesState.value; + if (events == null) return null; + final spoken = voiceConversation.update( + events: events, + members: membersState.value, + currentPubkey: currentPubkey, + ); + for (final event in spoken) { + ref + .read(pocketVoiceProvider.notifier) + .speak('channel:${channel.id}', event.content); + } + return null; + }, [voiceEventKey, membersState.value, currentPubkey, channel.id]); // Only show channel-level typing (exclude thread-scoped entries and self). final typingEntries = ref .watch(channelTypingProvider(channel.id)) @@ -254,6 +277,8 @@ class ChannelDetailPage extends HookConsumerWidget { ], ), actions: [ + if (!resolvedChannel.isForum) + PocketVoiceButton(conversationKey: 'channel:${channel.id}'), _MembersButton( channelId: resolvedChannel.id, channel: resolvedChannel, diff --git a/mobile/lib/features/channels/pocket_voice_button.dart b/mobile/lib/features/channels/pocket_voice_button.dart new file mode 100644 index 0000000000..dc8571cbe1 --- /dev/null +++ b/mobile/lib/features/channels/pocket_voice_button.dart @@ -0,0 +1,56 @@ +import 'package:flutter/material.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:lucide_icons_flutter/lucide_icons.dart'; + +import '../../shared/theme/theme.dart'; +import '../../shared/voice/pocket_model_provider.dart'; +import '../../shared/voice/pocket_voice_controller.dart'; + +class PocketVoiceButton extends ConsumerWidget { + const PocketVoiceButton({super.key, required this.conversationKey}); + + final String conversationKey; + + @override + Widget build(BuildContext context, WidgetRef ref) { + final voice = ref.watch(pocketVoiceProvider); + final model = ref.watch(pocketModelProvider); + final active = voice.enabled && voice.conversationKey == conversationKey; + + return IconButton( + color: active ? context.colors.primary : null, + tooltip: active ? 'Stop Pocket voice' : 'Start Pocket voice', + onPressed: () async { + if (active) { + await ref.read(pocketVoiceProvider.notifier).disable(); + return; + } + if (model.phase != PocketModelPhase.ready) { + if (context.mounted) { + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar( + content: Text('Download Pocket voice in Settings first.'), + ), + ); + } + return; + } + try { + await ref.read(pocketVoiceProvider.notifier).enable(conversationKey); + } catch (error) { + if (context.mounted) { + ScaffoldMessenger.of( + context, + ).showSnackBar(SnackBar(content: Text(error.toString()))); + } + } + }, + icon: voice.phase == PocketVoicePhase.loading && active + ? const SizedBox.square( + dimension: 18, + child: CircularProgressIndicator(strokeWidth: 2), + ) + : const Icon(LucideIcons.audioLines, size: 21), + ); + } +} diff --git a/mobile/lib/features/channels/pocket_voice_conversation.dart b/mobile/lib/features/channels/pocket_voice_conversation.dart new file mode 100644 index 0000000000..bdde9fbe57 --- /dev/null +++ b/mobile/lib/features/channels/pocket_voice_conversation.dart @@ -0,0 +1,51 @@ +import '../../shared/relay/relay.dart'; +import 'channel_management_provider.dart'; + +/// Selects newly-arrived bot messages for Pocket voice. +/// +/// Membership is authoritative: messages are never spoken until the channel's +/// member list has resolved, and history present when the view opens is only +/// used to establish the deduplication baseline. +class PocketVoiceConversation { + final Set _seen = {}; + bool _initialized = false; + + List update({ + required List events, + required List? members, + required String? currentPubkey, + String? threadHeadId, + }) { + final unseen = []; + for (final event in events) { + if (_seen.add(event.id)) unseen.add(event); + } + if (!_initialized) { + _initialized = true; + return const []; + } + if (members == null) return const []; + + final bots = { + for (final member in members) + if (member.isBot) member.pubkey.toLowerCase(), + }; + final self = currentPubkey?.toLowerCase(); + return [ + for (final event in unseen) + if ((event.kind == EventKind.streamMessage || + event.kind == EventKind.streamMessageV2) && + event.pubkey.toLowerCase() != self && + bots.contains(event.pubkey.toLowerCase()) && + event.content.trim().length > 1 && + !event.content.trimLeft().startsWith('[System]') && + _belongsToConversation(event, threadHeadId)) + event, + ]; + } + + static bool _belongsToConversation(NostrEvent event, String? threadHeadId) { + final parentId = event.threadReference.parentId; + return threadHeadId == null ? parentId == null : parentId == threadHeadId; + } +} diff --git a/mobile/lib/features/channels/send_message_provider.dart b/mobile/lib/features/channels/send_message_provider.dart index 3659bda4bf..fa37d3cae8 100644 --- a/mobile/lib/features/channels/send_message_provider.dart +++ b/mobile/lib/features/channels/send_message_provider.dart @@ -1,6 +1,7 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; import '../../shared/relay/relay.dart'; +import '../../shared/voice/pocket_voice_controller.dart'; import '../channels/channel_management_provider.dart'; import '../profile/user_cache_provider.dart'; import '../profile/user_profile.dart'; @@ -15,6 +16,7 @@ class SendMessage { final void Function(String channelId, NostrEvent event) _addLocalMessage; final void Function(String channelId, String eventId) _completeLocalMessage; final void Function(String channelId, String eventId) _removeLocalMessage; + final Future Function()? _onBeforeSend; SendMessage({ required SignedEventRelay signedEventRelay, @@ -25,12 +27,14 @@ class SendMessage { required void Function(String channelId, String eventId) completeLocalMessage, required void Function(String channelId, String eventId) removeLocalMessage, + Future Function()? onBeforeSend, }) : _signedEventRelay = signedEventRelay, _fetchMembers = fetchMembers, _readUserCache = readUserCache, _addLocalMessage = addLocalMessage, _completeLocalMessage = completeLocalMessage, - _removeLocalMessage = removeLocalMessage; + _removeLocalMessage = removeLocalMessage, + _onBeforeSend = onBeforeSend; /// Send a text message to a channel. /// @@ -47,6 +51,7 @@ class SendMessage { List? mentionPubkeys, List> mediaTags = const [], }) async { + await _onBeforeSend?.call(); // Use explicitly passed pubkeys, or resolve @mentions against // channel members to avoid matching the wrong user. final resolvedMentions = @@ -179,5 +184,10 @@ final sendMessageProvider = Provider((ref) { removeLocalMessage: (channelId, eventId) => ref .read(channelMessagesProvider(channelId).notifier) .removeLocalMessage(eventId), + onBeforeSend: () async { + if (ref.read(pocketVoiceProvider).enabled) { + await ref.read(pocketVoiceProvider.notifier).interrupt(); + } + }, ); }); diff --git a/mobile/lib/features/channels/thread_detail_page.dart b/mobile/lib/features/channels/thread_detail_page.dart index d00973b4d1..2e1898e022 100644 --- a/mobile/lib/features/channels/thread_detail_page.dart +++ b/mobile/lib/features/channels/thread_detail_page.dart @@ -7,9 +7,11 @@ import '../../shared/theme/theme.dart'; import '../../shared/widgets/avatar_image.dart'; import '../../shared/widgets/frosted_app_bar.dart'; import '../../shared/widgets/frosted_scaffold.dart'; +import '../../shared/voice/pocket_voice_controller.dart'; import '../profile/user_cache_provider.dart'; import '../profile/user_profile.dart'; import 'channel_link_navigation.dart'; +import 'channel_management_provider.dart'; import 'channel_typing_provider.dart'; import 'thread_replies_provider.dart'; import 'channels_provider.dart'; @@ -20,6 +22,8 @@ import '../profile/user_profile_sheet.dart'; import 'message_actions.dart'; import 'message_content.dart'; import 'mentions/mention_candidates_provider.dart'; +import 'pocket_voice_button.dart'; +import 'pocket_voice_conversation.dart'; import 'reaction_row.dart'; import 'read_state/read_state_format.dart'; import 'read_state/read_state_provider.dart'; @@ -67,6 +71,35 @@ class ThreadDetailPage extends HookConsumerWidget { }); final fetchedReplies = replyMessages.value; + final membersState = ref.watch(channelMembersProvider(channelId)); + final voiceConversation = useMemoized(PocketVoiceConversation.new); + final voiceEventKey = + repliesState.value?.map((event) => event.id).join(',') ?? ''; + useEffect( + () { + final events = repliesState.value; + if (events == null) return null; + final spoken = voiceConversation.update( + events: events, + members: membersState.value, + currentPubkey: currentPubkey, + threadHeadId: threadHead.id, + ); + for (final event in spoken) { + ref + .read(pocketVoiceProvider.notifier) + .speak('thread:$channelId:${threadHead.id}', event.content); + } + return null; + }, + [ + voiceEventKey, + membersState.value, + currentPubkey, + channelId, + threadHead.id, + ], + ); final allMsgs = fetchedReplies == null ? allMessages : [ @@ -153,7 +186,14 @@ class ThreadDetailPage extends HookConsumerWidget { }); return FrostedScaffold( - appBar: const FrostedAppBar(title: Text('Thread')), + appBar: FrostedAppBar( + title: const Text('Thread'), + actions: [ + PocketVoiceButton( + conversationKey: 'thread:$channelId:${threadHead.id}', + ), + ], + ), body: Column( children: [ Expanded( diff --git a/mobile/lib/features/settings/settings_page.dart b/mobile/lib/features/settings/settings_page.dart index f089f63915..289c3a1497 100644 --- a/mobile/lib/features/settings/settings_page.dart +++ b/mobile/lib/features/settings/settings_page.dart @@ -1,3 +1,5 @@ +import 'dart:async'; + import 'package:flutter/material.dart'; import 'package:flutter_hooks/flutter_hooks.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; @@ -9,6 +11,9 @@ import '../../shared/auth/auth.dart'; import '../../shared/clipboard_utils.dart'; import '../../shared/relay/relay.dart'; import '../../shared/theme/theme.dart'; +import '../../shared/voice/pocket_model_manifest.dart'; +import '../../shared/voice/pocket_model_provider.dart'; +import '../../shared/voice/pocket_voice_controller.dart'; import '../../shared/widgets/app_list.dart'; import '../../shared/widgets/app_list_card.dart'; import '../../shared/widgets/frosted_app_bar.dart'; @@ -18,6 +23,7 @@ import 'theme_picker_page.dart'; part 'settings_page/appearance_section.dart'; part 'settings_page/connection_section.dart'; +part 'settings_page/voice_section.dart'; class SettingsPage extends HookConsumerWidget { const SettingsPage({super.key, required this.profileHeader}); @@ -42,6 +48,7 @@ class SettingsPage extends HookConsumerWidget { children: [ profileHeader, const _AppearanceSection(), + const _VoiceSection(), const _ConnectionSection(), const _RemoveCommunitySection(), ], diff --git a/mobile/lib/features/settings/settings_page/voice_section.dart b/mobile/lib/features/settings/settings_page/voice_section.dart new file mode 100644 index 0000000000..e769e29d37 --- /dev/null +++ b/mobile/lib/features/settings/settings_page/voice_section.dart @@ -0,0 +1,130 @@ +part of '../settings_page.dart'; + +class _VoiceSection extends ConsumerWidget { + const _VoiceSection(); + + @override + Widget build(BuildContext context, WidgetRef ref) { + final model = ref.watch(pocketModelProvider); + final downloading = model.phase == PocketModelPhase.downloading; + final switching = + downloading || + model.phase == PocketModelPhase.checking || + model.phase == PocketModelPhase.verifying; + final status = switch (model.phase) { + PocketModelPhase.checking => 'Checking…', + PocketModelPhase.absent => 'Not downloaded', + PocketModelPhase.downloading => + '${(model.progress * 100).clamp(0, 100).round()}%', + PocketModelPhase.verifying => 'Verifying…', + PocketModelPhase.ready => 'Ready', + PocketModelPhase.cancelled => 'Cancelled', + PocketModelPhase.insufficientSpace => 'Needs more space', + PocketModelPhase.error => 'Download failed', + }; + final action = switch (model.phase) { + PocketModelPhase.ready || + PocketModelPhase.checking || + PocketModelPhase.verifying => null, + PocketModelPhase.downloading => + () => ref.read(pocketModelProvider.notifier).cancel(), + _ => () => ref.read(pocketModelProvider.notifier).download(), + }; + + return AppListCard( + label: 'Voice', + children: [ + AppListRow( + icon: LucideIcons.gauge, + title: 'Model size', + subtitle: model.variant == PocketModelVariant.higherQuality + ? 'Best voice quality. Uses more storage and memory.' + : 'Smaller and faster. Uses the supported hybrid INT8 model.', + subtitleMaxLines: 2, + value: model.variant.label, + trailing: switching ? null : const _RowChevron(), + onTap: switching + ? null + : () => _showModelSizePicker(context, ref, model.variant), + ), + AppListRow( + icon: LucideIcons.audioLines, + title: 'Pocket voice', + subtitle: + model.message ?? + 'Private, on-device speech. Downloads ' + '${model.variant.runtimeBytes ~/ 1000000} MB after install.', + subtitleMaxLines: 3, + value: status, + trailing: downloading + ? SizedBox.square( + dimension: 20, + child: CircularProgressIndicator( + value: model.total == 0 ? null : model.progress, + strokeWidth: 2, + ), + ) + : action == null + ? null + : Icon( + downloading ? LucideIcons.x : LucideIcons.download, + size: 18, + color: context.colors.primary, + ), + onTap: action, + ), + AppListRow( + icon: LucideIcons.hardDriveDownload, + title: 'Model storage', + value: '${model.variant.runtimeBytes ~/ 1000000} MB', + subtitle: 'Stored in application support, not duplicated in assets.', + subtitleMaxLines: 2, + ), + ], + ); + } +} + +Future _showModelSizePicker( + BuildContext context, + WidgetRef ref, + PocketModelVariant selected, +) => showModalBottomSheet( + context: context, + showDragHandle: true, + builder: (context) => SafeArea( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + for (final variant in PocketModelVariant.values) + ListTile( + leading: Icon( + variant == selected + ? LucideIcons.circleCheck + : LucideIcons.circle, + ), + title: Text(variant.label), + subtitle: Text( + variant == PocketModelVariant.higherQuality + ? 'FP32 · ${variant.runtimeBytes ~/ 1000000} MB' + : 'Hybrid INT8 · ${variant.runtimeBytes ~/ 1000000} MB', + ), + onTap: () { + Navigator.pop(context); + unawaited(_selectPocketModelVariant(ref, variant)); + }, + ), + const SizedBox(height: Grid.xs), + ], + ), + ), +); + +Future _selectPocketModelVariant( + WidgetRef ref, + PocketModelVariant variant, +) async { + if (variant == ref.read(pocketModelProvider).variant) return; + await ref.read(pocketVoiceProvider.notifier).releaseEngineForModelSelection(); + await ref.read(pocketModelProvider.notifier).select(variant); +} diff --git a/mobile/lib/shared/voice/pocket_model_downloader.dart b/mobile/lib/shared/voice/pocket_model_downloader.dart new file mode 100644 index 0000000000..bfd1641df0 --- /dev/null +++ b/mobile/lib/shared/voice/pocket_model_downloader.dart @@ -0,0 +1,558 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +import 'package:flutter/services.dart'; +import 'package:http/http.dart' as http; +import 'package:path_provider/path_provider.dart'; +import 'package:pointycastle/digests/sha256.dart'; +import 'package:uuid/uuid.dart'; + +import 'pocket_model_manifest.dart'; + +const _manifestName = '.buzz-model-manifest'; +const _legacyJanuaryModelVersion = '3'; +const _storageChannel = MethodChannel('buzz/voice_audio'); +const _minimumReserveBytes = 64 * 1024 * 1024; +const _defaultSendTimeout = Duration(seconds: 30); +const _defaultBodyIdleTimeout = Duration(seconds: 30); + +class PocketDownloadCancelled implements Exception { + const PocketDownloadCancelled(); +} + +class PocketInsufficientSpace implements Exception { + final int required; + final int available; + + const PocketInsufficientSpace(this.required, this.available); +} + +class PocketModelDownloadException implements Exception { + final String message; + final bool retryable; + + const PocketModelDownloadException(this.message, {this.retryable = false}); + + @override + String toString() => message; +} + +typedef PocketDownloadProgress = + void Function(int downloaded, int total, String currentFile); + +class PocketModelDownloader { + final http.Client Function() clientFactory; + final Future Function() applicationSupportDirectory; + final Future Function(String path) availableCapacity; + final Future Function(String path) excludeFromBackup; + final PocketModelVariant variant; + final List artifacts; + final String installationId; + final Duration sendTimeout; + final Duration bodyIdleTimeout; + + http.Client? _client; + bool _cancelled = false; + + PocketModelDownloader({ + http.Client Function()? clientFactory, + Future Function()? applicationSupportDirectory, + Future Function(String path)? availableCapacity, + Future Function(String path)? excludeFromBackup, + this.variant = PocketModelVariant.higherQuality, + this.artifacts = const [], + String? installationId, + this.sendTimeout = _defaultSendTimeout, + this.bodyIdleTimeout = _defaultBodyIdleTimeout, + }) : clientFactory = clientFactory ?? http.Client.new, + applicationSupportDirectory = + applicationSupportDirectory ?? getApplicationSupportDirectory, + availableCapacity = availableCapacity ?? _platformAvailableCapacity, + excludeFromBackup = excludeFromBackup ?? _platformExcludeFromBackup, + installationId = installationId ?? const Uuid().v4(); + + List get _artifacts => + artifacts.isEmpty ? pocketModelArtifacts[variant]! : artifacts; + + int get _downloadBytes => + _artifacts.fold(0, (total, artifact) => total + artifact.size); + + Future modelDirectory() async { + final support = await applicationSupportDirectory(); + return Directory( + '${support.path}/buzz/models/pocket-tts/' + 'v$pocketModelVersion/${variant.cacheKey}', + ); + } + + Future isReady() async { + final directory = await modelDirectory(); + if (await verify(directory, hashContents: false)) return true; + if (variant == PocketModelVariant.higherQuality && + await _migrateLegacyFp32(directory)) { + return true; + } + return false; + } + + Future verify(Directory directory, {required bool hashContents}) async { + final manifest = File('${directory.path}/$_manifestName'); + if (!await manifest.isFile()) return false; + try { + final decoded = + jsonDecode(await manifest.readAsString()) as Map; + if (decoded['version'] != pocketModelVersion || + decoded['revision'] != pocketModelRevision || + decoded['precision'] != variant.precision || + decoded['complete'] != true) { + return false; + } + for (final artifact in _artifacts) { + final file = File('${directory.path}/${artifact.name}'); + if (!await file.isFile() || await file.length() != artifact.size) { + return false; + } + if (hashContents && await _sha256File(file) != artifact.sha256) { + return false; + } + } + return File('${directory.path}/MODEL_LICENSE.txt').isFile(); + } catch (_) { + return false; + } + } + + Future install(PocketDownloadProgress onProgress) async { + _cancelled = false; + final finalDirectory = await modelDirectory(); + final root = finalDirectory.parent.parent; + await finalDirectory.parent.create(recursive: true); + await _cleanupAbandoned(root, finalDirectory.path); + + final available = await availableCapacity(root.path); + final reserve = max(_minimumReserveBytes, (_downloadBytes * 0.1).ceil()); + final required = _downloadBytes + reserve; + if (available < required) { + throw PocketInsufficientSpace(required, available); + } + + final staging = Directory( + '${root.path}/.pocket-tts-v$pocketModelVersion-' + '${variant.cacheKey}.install-$installationId', + ); + await staging.create(recursive: true); + await excludeFromBackup(staging.path); + var downloaded = 0; + _client = clientFactory(); + try { + for (final artifact in _artifacts) { + _throwIfCancelled(); + await _ensureCapacity(root.path, artifact.size); + await _downloadArtifact( + artifact, + staging, + (fileBytes) => + onProgress(downloaded + fileBytes, _downloadBytes, artifact.name), + ); + downloaded += artifact.size; + onProgress(downloaded, _downloadBytes, artifact.name); + } + + await File( + '${staging.path}/MODEL_LICENSE.txt', + ).writeAsString(pocketModelLicenseText, flush: true); + await _writeManifest(staging); + if (!await verify(staging, hashContents: true)) { + throw const PocketModelDownloadException( + 'Pocket model verification failed after download.', + ); + } + + final backup = Directory( + '${root.path}/.pocket-tts-v$pocketModelVersion-' + '${variant.cacheKey}.old-$installationId', + ); + if (await finalDirectory.exists()) { + await finalDirectory.rename(backup.path); + } + try { + await staging.rename(finalDirectory.path); + } catch (_) { + if (await backup.exists() && !await finalDirectory.exists()) { + await backup.rename(finalDirectory.path); + } + rethrow; + } + if (await backup.exists()) { + await backup.delete(recursive: true); + } + await excludeFromBackup(finalDirectory.path); + return finalDirectory; + } on PocketDownloadCancelled { + rethrow; + } finally { + _client?.close(); + _client = null; + if (await staging.exists()) { + await staging.delete(recursive: true); + } + } + } + + void cancel() { + _cancelled = true; + _client?.close(); + } + + Future _downloadArtifact( + PocketModelArtifact artifact, + Directory staging, + void Function(int fileBytes) onProgress, + ) async { + Object? lastError; + for (var attempt = 0; attempt < 3; attempt += 1) { + _throwIfCancelled(); + final part = File('${staging.path}/${artifact.name}.part'); + if (await part.exists()) await part.delete(); + try { + final request = http.Request('GET', artifact.url); + final response = await _client!.send(request).timeout(sendTimeout); + if (response.statusCode != HttpStatus.ok) { + final retryable = + response.statusCode == 408 || + response.statusCode == 429 || + response.statusCode >= 500; + throw PocketModelDownloadException( + '${artifact.name} download returned HTTP ' + '${response.statusCode}.', + retryable: retryable, + ); + } + if (response.contentLength case final length? + when length != artifact.size) { + throw PocketModelDownloadException( + '${artifact.name} expected ${artifact.size} bytes, ' + 'server reported $length.', + ); + } + + final digest = SHA256Digest(); + final sink = part.openWrite(); + var bytes = 0; + try { + await for (final chunk in response.stream.timeout(bodyIdleTimeout)) { + _throwIfCancelled(); + bytes += chunk.length; + if (bytes > artifact.size) { + throw PocketModelDownloadException( + '${artifact.name} exceeded ${artifact.size} bytes.', + ); + } + final typed = Uint8List.fromList(chunk); + digest.update(typed, 0, typed.length); + sink.add(chunk); + onProgress(bytes); + } + await sink.flush(); + } finally { + await sink.close(); + } + if (bytes != artifact.size) { + throw PocketModelDownloadException( + '${artifact.name} expected ${artifact.size} bytes, received $bytes.', + retryable: true, + ); + } + final output = Uint8List(digest.digestSize); + digest.doFinal(output, 0); + final hash = output + .map((byte) => byte.toRadixString(16).padLeft(2, '0')) + .join(); + if (hash != artifact.sha256) { + throw PocketModelDownloadException( + '${artifact.name} checksum did not match.', + ); + } + await part.rename('${staging.path}/${artifact.name}'); + return; + } on PocketDownloadCancelled { + rethrow; + } catch (error) { + _throwIfCancelled(); + lastError = error; + if (await part.exists()) await part.delete(); + final retryable = + error is SocketException || + error is TimeoutException || + (error is PocketModelDownloadException && error.retryable); + if (!retryable || attempt == 2) break; + if (error is TimeoutException || error is SocketException) { + _client?.close(); + _client = clientFactory(); + } + await Future.delayed(Duration(milliseconds: 250 << attempt)); + } + } + if (lastError is PocketModelDownloadException) throw lastError; + throw PocketModelDownloadException( + 'Unable to download ${artifact.name}: $lastError', + retryable: true, + ); + } + + Future _ensureCapacity(String path, int nextFileBytes) async { + final available = await availableCapacity(path); + final reserve = max(_minimumReserveBytes, (nextFileBytes * 0.1).ceil()); + if (available < nextFileBytes + reserve) { + throw PocketInsufficientSpace(nextFileBytes + reserve, available); + } + } + + Future _cleanupAbandoned(Directory root, String finalPath) async { + if (!await root.exists()) return; + final finalDirectory = Directory(finalPath); + final backups = []; + await for (final entity in root.list()) { + if (entity.path == finalPath) continue; + final name = entity.uri.pathSegments + .where((segment) => segment.isNotEmpty) + .last; + if (entity is! Directory) continue; + final prefix = '.pocket-tts-v$pocketModelVersion-${variant.cacheKey}.'; + if (name.startsWith('${prefix}old-')) { + backups.add(entity); + } else if (name.startsWith('${prefix}install-')) { + await entity.delete(recursive: true); + } + } + if (!await finalDirectory.exists() && backups.isNotEmpty) { + backups.sort( + (left, right) => + right.statSync().modified.compareTo(left.statSync().modified), + ); + await backups.removeAt(0).rename(finalPath); + } + for (final backup in backups) { + if (await backup.exists()) await backup.delete(recursive: true); + } + } + + Future deleteCache() async { + final directory = await modelDirectory(); + try { + if (await directory.exists()) await directory.delete(recursive: true); + } on FileSystemException { + // Cleanup is opportunistic. The selected verified cache remains usable. + } + if (variant == PocketModelVariant.higherQuality) { + await deleteLegacyCaches(); + } + } + + Future deleteLegacyCaches() async { + final directory = await modelDirectory(); + final root = directory.parent.parent; + try { + for (final version in [ + pocketLegacyFp32Version, + _legacyJanuaryModelVersion, + ]) { + final legacy = Directory('${root.path}/v$version'); + if (await legacy.exists()) await legacy.delete(recursive: true); + } + } on FileSystemException { + // A later readiness check retries cleanup. + } + } + + Future _migrateLegacyFp32(Directory finalDirectory) async { + final root = finalDirectory.parent.parent; + final legacy = Directory('${root.path}/v$pocketLegacyFp32Version'); + await _recoverLegacyFp32Backup(root, legacy); + final staged = await _legacyMigrationStaging(root); + if (!await finalDirectory.exists()) { + for (final candidate in staged.toList()) { + if (!await verify(candidate, hashContents: false)) continue; + await finalDirectory.parent.create(recursive: true); + await candidate.rename(finalDirectory.path); + staged.remove(candidate); + await excludeFromBackup(finalDirectory.path); + await _deleteOpportunistically( + Directory('${root.path}/v$_legacyJanuaryModelVersion'), + ); + for (final extra in staged) { + await _deleteOpportunistically(extra); + } + return true; + } + } + if (!await legacy.exists()) { + for (final candidate in staged.toList()) { + if (!await _verifyLegacyFp32(candidate)) continue; + await candidate.rename(legacy.path); + staged.remove(candidate); + break; + } + } + for (final extra in staged) { + await _deleteOpportunistically(extra); + } + + final staging = Directory( + '${root.path}/.pocket-tts-v$pocketModelVersion-fp32.' + 'migrate-$installationId', + ); + if (!await _verifyLegacyFp32(legacy)) return false; + + await finalDirectory.parent.create(recursive: true); + await legacy.rename(staging.path); + try { + await _writeManifest(staging); + if (!await verify(staging, hashContents: false)) { + throw const PocketModelDownloadException( + 'Migrated Pocket model verification failed.', + ); + } + await staging.rename(finalDirectory.path); + await excludeFromBackup(finalDirectory.path); + final january = Directory('${root.path}/v$_legacyJanuaryModelVersion'); + await _deleteOpportunistically(january); + return true; + } catch (_) { + if (await staging.exists() && !await legacy.exists()) { + await staging.rename(legacy.path); + } + rethrow; + } + } + + Future> _legacyMigrationStaging(Directory root) async { + if (!await root.exists()) return []; + final candidates = []; + final prefix = '.pocket-tts-v$pocketModelVersion-fp32.migrate-'; + await for (final entity in root.list()) { + if (entity is! Directory) continue; + final name = entity.uri.pathSegments + .where((segment) => segment.isNotEmpty) + .last; + if (name.startsWith(prefix)) candidates.add(entity); + } + candidates.sort( + (left, right) => + right.statSync().modified.compareTo(left.statSync().modified), + ); + return candidates; + } + + Future _recoverLegacyFp32Backup( + Directory root, + Directory legacy, + ) async { + if (!await root.exists()) return; + final backups = []; + await for (final entity in root.list()) { + if (entity is! Directory) continue; + final name = entity.uri.pathSegments + .where((segment) => segment.isNotEmpty) + .last; + if (name.startsWith('.pocket-tts-v$pocketLegacyFp32Version.old-')) { + backups.add(entity); + } + } + backups.sort( + (left, right) => + right.statSync().modified.compareTo(left.statSync().modified), + ); + if (!await legacy.exists()) { + for (final backup in backups.toList()) { + if (!await _verifyLegacyFp32(backup)) continue; + await backup.rename(legacy.path); + backups.remove(backup); + break; + } + } + for (final backup in backups) { + await _deleteOpportunistically(backup); + } + } + + Future _deleteOpportunistically(Directory directory) async { + try { + if (await directory.exists()) await directory.delete(recursive: true); + } on FileSystemException { + // A later readiness check retries stale-cache cleanup. + } + } + + Future _verifyLegacyFp32(Directory directory) async { + final manifest = File('${directory.path}/$_manifestName'); + if (!await manifest.isFile()) return false; + try { + final decoded = + jsonDecode(await manifest.readAsString()) as Map; + if (decoded['version'] != pocketLegacyFp32Version || + decoded['revision'] != pocketModelRevision || + decoded['precision'] != PocketModelVariant.higherQuality.precision || + decoded['complete'] != true) { + return false; + } + for (final artifact in _artifacts) { + final file = File('${directory.path}/${artifact.name}'); + if (!await file.isFile() || await file.length() != artifact.size) { + return false; + } + } + return File('${directory.path}/MODEL_LICENSE.txt').isFile(); + } catch (_) { + return false; + } + } + + Future _writeManifest(Directory directory) => + File('${directory.path}/$_manifestName').writeAsString( + jsonEncode({ + 'version': pocketModelVersion, + 'revision': pocketModelRevision, + 'precision': variant.precision, + 'complete': true, + 'artifacts': [ + for (final artifact in _artifacts) + { + 'name': artifact.name, + 'size': artifact.size, + 'sha256': artifact.sha256, + }, + ], + }), + flush: true, + ); + + void _throwIfCancelled() { + if (_cancelled) throw const PocketDownloadCancelled(); + } +} + +Future _sha256File(File file) async { + final digest = SHA256Digest(); + await for (final chunk in file.openRead()) { + final typed = Uint8List.fromList(chunk); + digest.update(typed, 0, typed.length); + } + final output = Uint8List(digest.digestSize); + digest.doFinal(output, 0); + return output.map((byte) => byte.toRadixString(16).padLeft(2, '0')).join(); +} + +Future _platformAvailableCapacity(String path) async => + await _storageChannel.invokeMethod('availableCapacity', path) ?? 0; + +Future _platformExcludeFromBackup(String path) => + _storageChannel.invokeMethod('excludeFromBackup', path); + +extension on File { + Future isFile() async => + await exists() && + await stat().then((stat) => stat.type == FileSystemEntityType.file); +} diff --git a/mobile/lib/shared/voice/pocket_model_manifest.dart b/mobile/lib/shared/voice/pocket_model_manifest.dart new file mode 100644 index 0000000000..ed40192f18 --- /dev/null +++ b/mobile/lib/shared/voice/pocket_model_manifest.dart @@ -0,0 +1,197 @@ +import 'package:flutter/foundation.dart'; + +enum PocketModelVariant { + higherQuality( + precision: 'fp32', + nativePrecision: 0, + label: 'Higher quality', + coreBytes: 439555904, + runtimeBytes: 440213643, + ), + faster( + precision: 'int8', + nativePrecision: 1, + label: 'Faster', + coreBytes: 165232420, + runtimeBytes: 165890159, + ); + + final String precision; + final int nativePrecision; + final String label; + final int coreBytes; + final int runtimeBytes; + + const PocketModelVariant({ + required this.precision, + required this.nativePrecision, + required this.label, + required this.coreBytes, + required this.runtimeBytes, + }); + + String get cacheKey => precision; + + static PocketModelVariant fromStored(String? value) => + values.where((variant) => variant.precision == value).firstOrNull ?? + higherQuality; +} + +@immutable +class PocketModelArtifact { + final String name; + final Uri url; + final int size; + final String sha256; + + const PocketModelArtifact({ + required this.name, + required this.url, + required this.size, + required this.sha256, + }); +} + +const pocketModelVersion = '5'; +const pocketLegacyFp32Version = '4'; +const pocketModelRevision = '58a6d00cf13d239b6748cb0769f35c580a8f606c'; +const _pocketModelBase = + 'https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/' + '$pocketModelRevision/onnx/english_2026-04'; +const _pocketLicenseUrl = + 'https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/' + '$pocketModelRevision/onnx/LICENSE'; +const _pocketVoiceUrl = + 'https://huggingface.co/kyutai/tts-voices/resolve/' + '323332d33f997de8394f24a193e1a76df720e01a/' + 'vctk/p333_023_enhanced.wav'; + +final pocketModelArtifacts = >{ + PocketModelVariant.higherQuality: [ + ..._commonArtifacts, + _artifact( + 'flow_lm_main.onnx', + 302742149, + '6d18315e2c33ca3e3aa4a4e3dca22f56d007fd823127e24948b37695bf54190f', + ), + _artifact( + 'flow_lm_flow.onnx', + 39097095, + '085d239f68897e28fb06e95c743738ad8b8c092ee6dc55f5491313e81ff08062', + ), + _artifact( + 'mimi_decoder.onnx', + 41471926, + '86f038caa02a96a0ff9c25526a0ff43a4906c418197ed72d3e30f720ac7ce802', + ), + ..._sharedFp32Graphs, + ..._voiceAndLicense, + ], + PocketModelVariant.faster: [ + ..._commonArtifacts, + _artifact( + 'flow_lm_main_int8.onnx', + 76341079, + 'f9bd8106b79a0192c1c43399ab938fb24900a95c1c599870d75a884e99000116', + ), + _artifact( + 'flow_lm_flow_int8.onnx', + 9962530, + '3dd781ee5abee9e195320bf0106bebd6372a852b3b36352524ee78b40554635d', + ), + _artifact( + 'mimi_decoder_int8.onnx', + 22684077, + '3630450a3297a101792a6ac66619ebc70ab916b265e6220c2afaef8b1673f925', + ), + ..._sharedFp32Graphs, + ..._voiceAndLicense, + ], +}; + +final _commonArtifacts = [ + _artifact( + 'bundle.json', + 24381, + 'bab643150f437f37df080a710520ff39ed9ebd9a339f8ebdc739f7eddfc28b3f', + ), + _artifact( + 'bos_before_voice.npy', + 4224, + 'f46edf4f7007b7ba4ea58831f49d003e59e167b4641c44bb3addfe9231a780b1', + ), + _artifact( + 'tokenizer.model', + 59339, + 'd461765ae179566678c93091c5fa6f2984c31bbe990bf1aa62d92c64d91bc3f6', + ), +]; + +final _sharedFp32Graphs = [ + _artifact( + 'mimi_encoder.onnx', + 39768446, + '853e2ca623b8782d94c3745ec6133bfdff7ce33d9b11128bd29ea03f28d76e3d', + ), + _artifact( + 'text_conditioner.onnx', + 16388344, + '4ecee995fb69f85c7a7493d11f7b5ee15d9950facc7ab3f5c9c49ef1e03847bb', + ), +]; + +final _voiceAndLicense = [ + PocketModelArtifact( + name: 'reference_sample.wav', + url: Uri.parse(_pocketVoiceUrl), + size: 639084, + sha256: 'a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f', + ), + PocketModelArtifact( + name: 'LICENSE', + url: Uri.parse(_pocketLicenseUrl), + size: 18655, + sha256: 'fe7b4ce83b8381cc5b216bbb4af73c570688d1b819c73bbaed8ca401f4677cd6', + ), +]; + +PocketModelArtifact _artifact(String name, int size, String sha256) => + PocketModelArtifact( + name: name, + url: Uri.parse('$_pocketModelBase/$name'), + size: size, + sha256: sha256, + ); + +const pocketModelLicenseText = ''' +Pocket TTS +© Kyutai. + +Licensed under the Creative Commons Attribution 4.0 International License +(CC-BY-4.0). License text: https://creativecommons.org/licenses/by/4.0/ + +Original model by Kyutai: https://huggingface.co/kyutai/pocket-tts +Paper: Charles, Roebel, et al., Pocket TTS (arXiv:2509.06926). +Mimi neural codec by Kyutai is bundled as part of the model. + +April 2026 ONNX export by KevinAHM: +https://huggingface.co/KevinAHM/pocket-tts-onnx +Pinned revision: 58a6d00cf13d239b6748cb0769f35c580a8f606c +Bundle: english_2026-04. The Faster option uses the upstream-supported +three-graph INT8 selection; encoder and text conditioner remain FP32. + +Bundled reference voice (reference_sample.wav): +"Mary (f, conversation)" preset from the Kyutai TTS demo voice catalogue +(https://kyutai.org/tts), distributed via +https://huggingface.co/kyutai/tts-voices as `vctk/p333_023_enhanced.wav`. +Original recording from the Voice Cloning Toolkit (VCTK) corpus, speaker p333: +https://datashare.ed.ac.uk/handle/10283/3443 (CC-BY-4.0). +Recording enhancement (denoise/dereverb) by ai-coustics: +https://ai-coustics.com/ + +Buzz ships all ONNX/model artifacts and the reference voice WAV unmodified, +renamed only by placement in the local model directory. + +Provided "AS IS", without warranty of any kind, express or implied. See the +license text for full warranty disclaimer. +'''; diff --git a/mobile/lib/shared/voice/pocket_model_provider.dart b/mobile/lib/shared/voice/pocket_model_provider.dart new file mode 100644 index 0000000000..186b2a8088 --- /dev/null +++ b/mobile/lib/shared/voice/pocket_model_provider.dart @@ -0,0 +1,297 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +import '../theme/theme_provider.dart'; +import 'pocket_model_downloader.dart'; +import 'pocket_model_manifest.dart'; +import 'pocket_voice_worker.dart'; + +const _pocketModelSelectionKey = 'pocket_model_precision'; + +enum PocketModelPhase { + checking, + absent, + downloading, + verifying, + ready, + cancelled, + insufficientSpace, + error, +} + +@immutable +class PocketModelState { + final PocketModelPhase phase; + final PocketModelVariant variant; + final int downloaded; + final int total; + final String? path; + final String? message; + + const PocketModelState({ + required this.phase, + this.variant = PocketModelVariant.higherQuality, + this.downloaded = 0, + this.total = 0, + this.path, + this.message, + }); + + double get progress => total == 0 ? 0 : downloaded / total; +} + +final pocketModelDownloaderProvider = + Provider.family( + (_, variant) => PocketModelDownloader(variant: variant), + ); + +typedef PocketModelValidator = + Future Function(String path, PocketModelVariant variant); + +final pocketModelValidatorProvider = Provider( + (_) => (path, variant) async { + final worker = PocketVoiceWorker(); + StreamSubscription? subscription; + try { + await worker.start(path, precision: variant.nativePrecision); + const generation = 1; + var hasNonSilentPcm = false; + final completed = Completer(); + subscription = worker.responses.listen((response) { + switch (response) { + case PocketWorkerAudio( + generation: generation, + :final data, + :final isLast, + ): + final bytes = data.materialize().asUint8List(); + hasNonSilentPcm = + hasNonSilentPcm || bytes.any((sample) => sample != 0); + if (isLast && !completed.isCompleted) { + if (hasNonSilentPcm) { + completed.complete(); + } else { + completed.completeError( + StateError('Pocket model produced only silent audio.'), + ); + } + } + case PocketWorkerDone(generation: generation): + if (!completed.isCompleted) { + completed.completeError( + StateError('Pocket model produced no audio.'), + ); + } + case PocketWorkerFailure(generation: generation, :final message): + if (!completed.isCompleted) { + completed.completeError(StateError(message)); + } + case _: + break; + } + }); + worker.synthesize(generation, 'Pocket voice is ready.'); + await completed.future.timeout(const Duration(minutes: 1)); + } finally { + await subscription?.cancel(); + await worker.dispose(); + } + }, +); + +final pocketModelProvider = + NotifierProvider( + PocketModelNotifier.new, + ); + +class PocketModelNotifier extends Notifier { + PocketModelDownloader? _active; + int _operationEpoch = 0; + + @override + PocketModelState build() { + ref.onDispose(() => _active?.cancel()); + final stored = ref + .read(savedPrefsProvider) + .getString(_pocketModelSelectionKey); + final variant = PocketModelVariant.fromStored(stored); + Future.microtask(check); + return PocketModelState(phase: PocketModelPhase.checking, variant: variant); + } + + Future check() async { + final epoch = ++_operationEpoch; + final selected = state.variant; + final selectedDownloader = ref.read( + pocketModelDownloaderProvider(selected), + ); + final selectedReady = await selectedDownloader.isReady(); + if (epoch != _operationEpoch) return; + if (selectedReady) { + final directory = await selectedDownloader.modelDirectory(); + final obsolete = selected == PocketModelVariant.higherQuality + ? PocketModelVariant.faster + : PocketModelVariant.higherQuality; + await ref.read(pocketModelDownloaderProvider(obsolete)).deleteCache(); + if (selected == PocketModelVariant.higherQuality) { + await selectedDownloader.deleteLegacyCaches(); + } + if (epoch != _operationEpoch) return; + state = PocketModelState( + phase: PocketModelPhase.ready, + variant: selected, + path: directory.path, + ); + return; + } + + final fallback = selected == PocketModelVariant.higherQuality + ? PocketModelVariant.faster + : PocketModelVariant.higherQuality; + final fallbackDownloader = ref.read( + pocketModelDownloaderProvider(fallback), + ); + final fallbackReady = await fallbackDownloader.isReady(); + if (epoch != _operationEpoch) return; + if (fallbackReady) { + final directory = await fallbackDownloader.modelDirectory(); + if (epoch != _operationEpoch) return; + await _persist(fallback); + await selectedDownloader.deleteCache(); + if (epoch != _operationEpoch) return; + state = PocketModelState( + phase: PocketModelPhase.ready, + variant: fallback, + path: directory.path, + message: '${selected.label} was unavailable. Using ${fallback.label}.', + ); + return; + } + + state = PocketModelState(phase: PocketModelPhase.absent, variant: selected); + } + + Future download() => select(state.variant); + + Future select(PocketModelVariant target) async { + if (_active != null) return; + if (state.phase == PocketModelPhase.ready && state.variant == target) { + return; + } + + _operationEpoch += 1; + final previous = state; + final downloader = ref.read(pocketModelDownloaderProvider(target)); + _active = downloader; + try { + Directory directory; + if (await downloader.isReady()) { + directory = await downloader.modelDirectory(); + } else { + state = PocketModelState( + phase: PocketModelPhase.downloading, + variant: target, + total: target.runtimeBytes, + ); + directory = await downloader.install((downloaded, total, _) { + state = PocketModelState( + phase: PocketModelPhase.downloading, + variant: target, + downloaded: downloaded, + total: total, + ); + }); + } + + state = PocketModelState( + phase: PocketModelPhase.verifying, + variant: target, + path: directory.path, + ); + await ref.read(pocketModelValidatorProvider)(directory.path, target); + await _persist(target); + state = PocketModelState( + phase: PocketModelPhase.ready, + variant: target, + path: directory.path, + ); + + final obsolete = target == PocketModelVariant.higherQuality + ? PocketModelVariant.faster + : PocketModelVariant.higherQuality; + await ref.read(pocketModelDownloaderProvider(obsolete)).deleteCache(); + if (target == PocketModelVariant.higherQuality) { + await downloader.deleteLegacyCaches(); + } + } on PocketDownloadCancelled { + await downloader.deleteCache(); + _restoreOrSet( + previous, + PocketModelState(phase: PocketModelPhase.cancelled, variant: target), + ); + } on PocketInsufficientSpace { + await downloader.deleteCache(); + _restoreOrSet( + previous, + PocketModelState( + phase: PocketModelPhase.insufficientSpace, + variant: target, + message: 'Not enough free space for ${target.label}.', + ), + ); + } on FileSystemException catch (error) { + await downloader.deleteCache(); + _restoreOrSet( + previous, + PocketModelState( + phase: PocketModelPhase.error, + variant: target, + message: error.osError?.errorCode == 28 + ? 'Pocket voice ran out of disk space.' + : 'Pocket voice storage failed: ${error.message}', + ), + ); + } catch (error) { + await downloader.deleteCache(); + _restoreOrSet( + previous, + PocketModelState( + phase: PocketModelPhase.error, + variant: target, + message: '${target.label} is unavailable: $error', + ), + ); + } finally { + _active = null; + } + } + + void cancel() => _active?.cancel(); + + void _restoreOrSet(PocketModelState previous, PocketModelState failure) { + if (previous.phase == PocketModelPhase.ready && previous.path != null) { + state = PocketModelState( + phase: PocketModelPhase.ready, + variant: previous.variant, + path: previous.path, + message: + '${failure.message ?? failure.phase.name} ' + '${previous.variant.label} was kept.', + ); + } else { + state = failure; + } + } + + Future _persist(PocketModelVariant variant) async { + final saved = await ref + .read(savedPrefsProvider) + .setString(_pocketModelSelectionKey, variant.precision); + if (!saved) { + throw StateError('Pocket model selection could not be saved.'); + } + } +} diff --git a/mobile/lib/shared/voice/pocket_voice_controller.dart b/mobile/lib/shared/voice/pocket_voice_controller.dart new file mode 100644 index 0000000000..31190d532e --- /dev/null +++ b/mobile/lib/shared/voice/pocket_voice_controller.dart @@ -0,0 +1,436 @@ +import 'dart:async'; +import 'dart:collection'; + +import 'package:flutter/foundation.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +import '../relay/relay.dart'; +import 'pocket_model_provider.dart'; +import 'pocket_voice_worker.dart'; +import 'voice_audio_output.dart'; + +enum PocketVoicePhase { off, loading, listening, synthesizing, speaking, error } + +enum PocketVoiceFailureKind { load, synthesis, playback } + +@immutable +class PocketVoiceState { + final PocketVoicePhase phase; + final String? conversationKey; + final PocketVoiceFailureKind? failureKind; + final String? error; + + const PocketVoiceState({ + this.phase = PocketVoicePhase.off, + this.conversationKey, + this.failureKind, + this.error, + }); + + bool get enabled => phase != PocketVoicePhase.off; +} + +final voiceAudioOutputProvider = Provider( + (_) => PlatformVoiceAudioOutput(), +); + +final pocketVoiceWorkerFactoryProvider = Provider( + (_) => PocketVoiceWorker.new, +); + +final pocketVoiceProvider = + NotifierProvider( + PocketVoiceNotifier.new, + ); + +class PocketVoiceNotifier extends Notifier { + final Queue _utterances = Queue(); + final Queue _audio = Queue(); + PocketVoiceWorker? _worker; + Future? _workerStart; + String? _workerStartingModelPath; + int? _workerStartingPrecision; + String? _workerModelPath; + int? _workerPrecision; + StreamSubscription? _workerSubscription; + StreamSubscription? _audioSubscription; + int _transitionEpoch = 0; + int _nextGeneration = 0; + int? _activeGeneration; + bool _synthesisComplete = false; + bool _playbackActive = false; + bool _stopping = false; + bool _queueWhileStopping = false; + Future? _stopFuture; + + @override + PocketVoiceState build() { + ref.listen(relayConfigProvider, (previous, _) { + if (previous != null) unawaited(disable()); + }); + ref.listen(pocketModelProvider, (_, model) { + if (model.phase == PocketModelPhase.ready && + _worker != null && + (_workerModelPath != model.path || + _workerPrecision != model.variant.nativePrecision)) { + unawaited(_replaceWorkerForModelChange()); + } + }); + _audioSubscription = ref + .read(voiceAudioOutputProvider) + .events + .listen(_handleAudioEvent); + ref.onDispose(() { + _transitionEpoch += 1; + _workerSubscription?.cancel(); + _audioSubscription?.cancel(); + _worker?.cancel(); + unawaited(_worker?.dispose()); + }); + return const PocketVoiceState(); + } + + Future enable(String conversationKey) async { + if (state.conversationKey == conversationKey && state.enabled) return; + final epoch = ++_transitionEpoch; + await _stopConversation(preserveIncoming: false); + if (epoch != _transitionEpoch) return; + + final model = ref.read(pocketModelProvider); + if (model.phase != PocketModelPhase.ready || model.path == null) { + throw StateError('Download Pocket voice before starting a conversation.'); + } + state = PocketVoiceState( + phase: PocketVoicePhase.loading, + conversationKey: conversationKey, + ); + try { + await _ensureWorker(model.path!, model.variant.nativePrecision); + if (epoch != _transitionEpoch) return; + state = PocketVoiceState( + phase: PocketVoicePhase.listening, + conversationKey: conversationKey, + ); + _startNextUtterance(); + } catch (error) { + if (epoch == _transitionEpoch) { + state = PocketVoiceState( + phase: PocketVoicePhase.error, + conversationKey: conversationKey, + failureKind: PocketVoiceFailureKind.load, + error: error.toString(), + ); + } + rethrow; + } + } + + Future disable() async { + _transitionEpoch += 1; + state = const PocketVoiceState(); + await _stopConversation(preserveIncoming: false); + } + + /// Stops playback and releases the resident engine before model validation. + Future releaseEngineForModelSelection() async { + await disable(); + final starting = _workerStart; + if (starting != null) { + try { + await starting; + } catch (_) { + // Selection validation reports the candidate engine's own result. + } + } + await _disposeWorker(); + } + + void speak(String conversationKey, String text) { + if (!state.enabled || state.conversationKey != conversationKey) return; + if (state.phase == PocketVoicePhase.error) return; + if (_stopping && !_queueWhileStopping) return; + final trimmed = text.trim(); + if (trimmed.length <= 1 || trimmed.startsWith('[System]')) return; + _utterances.add(trimmed); + _startNextUtterance(); + } + + Future interrupt() async { + final epoch = ++_transitionEpoch; + await _stopConversation(preserveIncoming: true); + if (epoch == _transitionEpoch && state.enabled) { + state = PocketVoiceState( + phase: PocketVoicePhase.listening, + conversationKey: state.conversationKey, + ); + _startNextUtterance(); + } + } + + Future _ensureWorker( + String modelPath, + int precision, + ) async { + final starting = _workerStart; + if (starting != null && + _workerStartingModelPath == modelPath && + _workerStartingPrecision == precision) { + return starting; + } + if (starting != null) { + try { + await starting; + } catch (_) { + // The replacement attempt below reports its own startup result. + } + } + + final worker = _worker; + if (worker != null && + worker.isReady && + _workerModelPath == modelPath && + _workerPrecision == precision) { + return worker; + } + if (worker != null) { + await _disposeWorker(); + } + + final created = ref.read(pocketVoiceWorkerFactoryProvider)(); + _worker = created; + _workerSubscription = created.responses.listen(_handleWorkerResponse); + final future = created + .start(modelPath, precision: precision) + .then((_) => created); + _workerStart = future; + _workerStartingModelPath = modelPath; + _workerStartingPrecision = precision; + try { + final ready = await future; + _workerModelPath = modelPath; + _workerPrecision = precision; + return ready; + } catch (error, stackTrace) { + if (identical(_worker, created)) { + _worker = null; + await _workerSubscription?.cancel(); + _workerSubscription = null; + } + await created.dispose(); + Error.throwWithStackTrace(error, stackTrace); + } finally { + if (identical(_workerStart, future)) { + _workerStart = null; + _workerStartingModelPath = null; + _workerStartingPrecision = null; + } + } + } + + Future _replaceWorkerForModelChange() async { + await releaseEngineForModelSelection(); + } + + Future _disposeWorker() async { + final worker = _worker; + _worker = null; + _workerModelPath = null; + _workerPrecision = null; + await _workerSubscription?.cancel(); + _workerSubscription = null; + if (worker != null) await worker.dispose(); + } + + Future _stopConversation({required bool preserveIncoming}) async { + final activeStop = _stopFuture; + if (activeStop != null) { + if (!preserveIncoming) { + _queueWhileStopping = false; + _utterances.clear(); + } + await activeStop; + return; + } + _stopping = true; + _queueWhileStopping = preserveIncoming; + _utterances.clear(); + _audio.clear(); + _activeGeneration = null; + _synthesisComplete = false; + _playbackActive = false; + final output = ref.read(voiceAudioOutputProvider); + final worker = _worker; + worker?.cancel(); + final stop = Future.wait([ + output.stop(), + if (worker != null) worker.cancelAndWait(), + ]); + _stopFuture = stop; + try { + await stop; + } finally { + if (identical(_stopFuture, stop)) { + _stopFuture = null; + _stopping = false; + _queueWhileStopping = false; + } + } + } + + void _startNextUtterance() { + if (_stopping || + _activeGeneration != null || + _utterances.isEmpty || + !state.enabled) { + return; + } + final worker = _worker; + if (worker == null || !worker.isReady) return; + final utterance = _utterances.removeFirst(); + final generation = ++_nextGeneration; + _activeGeneration = generation; + _synthesisComplete = false; + state = PocketVoiceState( + phase: PocketVoicePhase.synthesizing, + conversationKey: state.conversationKey, + ); + try { + worker.synthesize(generation, utterance); + } catch (error) { + _activeGeneration = null; + state = PocketVoiceState( + phase: PocketVoicePhase.error, + conversationKey: state.conversationKey, + failureKind: PocketVoiceFailureKind.synthesis, + error: error.toString(), + ); + } + } + + void _handleWorkerResponse(PocketWorkerResponse response) { + switch (response) { + case PocketWorkerReady(): + case PocketWorkerStopped(): + return; + case PocketWorkerDone(): + if (response.generation != _activeGeneration) return; + _synthesisComplete = true; + if (!_playbackActive && _audio.isEmpty) _finishUtterance(); + case PocketWorkerFailure(): + if (response.generation != _activeGeneration) return; + if (response.kind == PocketWorkerFailureKind.cancelled) { + _finishUtterance(); + return; + } + _failPlayback( + response.message, + failureKind: PocketVoiceFailureKind.synthesis, + ); + case PocketWorkerAudio(): + if (response.generation != _activeGeneration) return; + _audio.add(response); + if (response.isLast) _synthesisComplete = true; + unawaited(_playNextChunk()); + } + } + + Future _playNextChunk() async { + if (_playbackActive || _audio.isEmpty || _activeGeneration == null) return; + final activeGeneration = _activeGeneration; + final conversationKey = state.conversationKey; + final chunk = _audio.removeFirst(); + _playbackActive = true; + final output = ref.read(voiceAudioOutputProvider); + try { + await output.play( + chunk.data.materialize().asUint8List(), + chunk.sampleRate, + ); + } catch (error) { + if (activeGeneration == _activeGeneration && state.enabled) { + _failPlayback( + error.toString(), + failureKind: PocketVoiceFailureKind.playback, + ); + } + return; + } + if (activeGeneration != _activeGeneration || + !state.enabled || + state.conversationKey != conversationKey) { + _playbackActive = false; + await output.stop(); + return; + } + } + + void _handleAudioEvent(VoiceAudioEvent event) { + switch (event) { + case VoiceAudioEvent.started: + if (!_playbackActive || !state.enabled) return; + state = PocketVoiceState( + phase: PocketVoicePhase.speaking, + conversationKey: state.conversationKey, + ); + case VoiceAudioEvent.completed: + if (!_playbackActive) return; + _playbackActive = false; + if (_audio.isNotEmpty) { + unawaited(_playNextChunk()); + } else if (_synthesisComplete) { + _finishUtterance(); + } else if (state.enabled) { + state = PocketVoiceState( + phase: PocketVoicePhase.synthesizing, + conversationKey: state.conversationKey, + ); + } + case VoiceAudioEvent.error: + if (_playbackActive) { + _failPlayback( + 'Pocket voice playback failed.', + failureKind: PocketVoiceFailureKind.playback, + ); + } + case VoiceAudioEvent.interrupted: + case VoiceAudioEvent.routeLost: + case VoiceAudioEvent.backgrounded: + unawaited(interrupt()); + } + } + + void _failPlayback( + String message, { + required PocketVoiceFailureKind failureKind, + }) { + _worker?.cancel(); + _utterances.clear(); + _activeGeneration = null; + _audio.clear(); + _synthesisComplete = false; + _playbackActive = false; + unawaited(ref.read(voiceAudioOutputProvider).stop()); + if (state.enabled) { + state = PocketVoiceState( + phase: PocketVoicePhase.error, + conversationKey: state.conversationKey, + failureKind: failureKind, + error: message, + ); + } + } + + void _finishUtterance() { + _activeGeneration = null; + _synthesisComplete = false; + _playbackActive = false; + if (_utterances.isNotEmpty) { + _startNextUtterance(); + } else if (state.enabled) { + state = PocketVoiceState( + phase: PocketVoicePhase.listening, + conversationKey: state.conversationKey, + ); + } + } +} diff --git a/mobile/lib/shared/voice/pocket_voice_ffi.dart b/mobile/lib/shared/voice/pocket_voice_ffi.dart new file mode 100644 index 0000000000..18270d3314 --- /dev/null +++ b/mobile/lib/shared/voice/pocket_voice_ffi.dart @@ -0,0 +1,164 @@ +import 'dart:convert'; +import 'dart:ffi' as ffi; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; + +final class _EngineResult extends ffi.Struct { + external ffi.Pointer engine; + external ffi.Pointer error; +} + +final class _PcmResult extends ffi.Struct { + external ffi.Pointer samples; + + @ffi.IntPtr() + external int len; + + @ffi.Uint32() + external int sampleRate; + + external ffi.Pointer error; +} + +typedef _CreateNative = _EngineResult Function(ffi.Pointer, ffi.Uint8); +typedef _CreateDart = _EngineResult Function(ffi.Pointer, int); +typedef _SynthNative = + _PcmResult Function(ffi.Pointer, ffi.Pointer); +typedef _SynthDart = + _PcmResult Function(ffi.Pointer, ffi.Pointer); +typedef _CancelNative = ffi.Void Function(ffi.Pointer); +typedef _CancelDart = void Function(ffi.Pointer); +typedef _ResetCancelNative = ffi.Void Function(ffi.Pointer); +typedef _ResetCancelDart = void Function(ffi.Pointer); +typedef _PrepareNative = + ffi.Pointer Function(ffi.Pointer, ffi.Pointer); +typedef _PrepareDart = + ffi.Pointer Function(ffi.Pointer, ffi.Pointer); +typedef _DestroyNative = ffi.Void Function(ffi.Pointer); +typedef _DestroyDart = void Function(ffi.Pointer); +typedef _FreePcmNative = ffi.Void Function(_PcmResult); +typedef _FreePcmDart = void Function(_PcmResult); +typedef _FreeStringNative = ffi.Void Function(ffi.Pointer); +typedef _FreeStringDart = void Function(ffi.Pointer); + +class PocketVoicePcm { + final Uint8List bytes; + final int sampleRate; + + const PocketVoicePcm(this.bytes, this.sampleRate); +} + +class PocketVoiceFfi { + final ffi.DynamicLibrary _library; + late final _CreateDart _create; + late final _SynthDart _synthesize; + late final _CancelDart _cancel; + late final _ResetCancelDart _resetCancel; + late final _PrepareDart _prepare; + late final _DestroyDart _destroy; + late final _FreePcmDart _freePcm; + late final _FreeStringDart _freeString; + + PocketVoiceFfi() : _library = _openLibrary() { + _create = _library.lookupFunction<_CreateNative, _CreateDart>( + 'buzz_voice_engine_create', + ); + _synthesize = _library.lookupFunction<_SynthNative, _SynthDart>( + 'buzz_voice_engine_synthesize', + ); + _cancel = _library.lookupFunction<_CancelNative, _CancelDart>( + 'buzz_voice_engine_cancel', + ); + _resetCancel = _library + .lookupFunction<_ResetCancelNative, _ResetCancelDart>( + 'buzz_voice_engine_reset_cancel', + ); + _prepare = _library.lookupFunction<_PrepareNative, _PrepareDart>( + 'buzz_voice_prepare_chunks_json', + ); + _destroy = _library.lookupFunction<_DestroyNative, _DestroyDart>( + 'buzz_voice_engine_destroy', + ); + _freePcm = _library.lookupFunction<_FreePcmNative, _FreePcmDart>( + 'buzz_voice_pcm_free', + ); + _freeString = _library.lookupFunction<_FreeStringNative, _FreeStringDart>( + 'buzz_voice_string_free', + ); + } + + int create(String modelPath, int precision) { + final path = modelPath.toNativeUtf8(); + final result = _create(path, precision); + calloc.free(path); + if (result.error != ffi.nullptr) { + throw StateError(_takeError(result.error)); + } + if (result.engine == ffi.nullptr) { + throw StateError('Pocket engine creation returned a null handle.'); + } + return result.engine.address; + } + + PocketVoicePcm synthesize(int handle, String text) { + final input = text.toNativeUtf8(); + final result = _synthesize( + ffi.Pointer.fromAddress(handle), + input, + ); + calloc.free(input); + try { + if (result.error != ffi.nullptr) { + throw StateError(result.error.cast().toDartString()); + } + if (result.samples == ffi.nullptr) { + throw StateError('Pocket synthesis returned a null sample buffer.'); + } + final sampleBytes = result.samples.cast().asTypedList( + result.len * 2, + ); + return PocketVoicePcm(Uint8List.fromList(sampleBytes), result.sampleRate); + } finally { + _freePcm(result); + } + } + + void cancel(int handle) => _cancel(ffi.Pointer.fromAddress(handle)); + + void resetCancel(int handle) => + _resetCancel(ffi.Pointer.fromAddress(handle)); + + List prepareChunks(int handle, String text) { + final input = text.toNativeUtf8(); + final result = _prepare(ffi.Pointer.fromAddress(handle), input); + calloc.free(input); + if (result == ffi.nullptr) { + throw StateError('Pocket text preparation returned no result.'); + } + final encoded = _takeError(result); + final decoded = jsonDecode(encoded); + if (decoded is! List) { + throw StateError('Pocket text preparation returned invalid JSON.'); + } + return decoded.cast(); + } + + void destroy(int handle) => + _destroy(ffi.Pointer.fromAddress(handle)); + + String _takeError(ffi.Pointer error) { + final message = error.cast().toDartString(); + _freeString(error); + return message; + } +} + +ffi.DynamicLibrary _openLibrary() { + if (Platform.isIOS) return ffi.DynamicLibrary.process(); + if (Platform.isAndroid) { + return ffi.DynamicLibrary.open('libbuzz_voice_mobile.so'); + } + throw UnsupportedError('Pocket voice is supported only on iOS and Android.'); +} diff --git a/mobile/lib/shared/voice/pocket_voice_worker.dart b/mobile/lib/shared/voice/pocket_voice_worker.dart new file mode 100644 index 0000000000..eb573fd79f --- /dev/null +++ b/mobile/lib/shared/voice/pocket_voice_worker.dart @@ -0,0 +1,240 @@ +import 'dart:async'; +import 'dart:isolate'; + +import 'pocket_voice_ffi.dart'; + +sealed class PocketWorkerResponse { + const PocketWorkerResponse(); +} + +class PocketWorkerReady extends PocketWorkerResponse { + final int handle; + + const PocketWorkerReady(this.handle); +} + +class PocketWorkerAudio extends PocketWorkerResponse { + final int generation; + final TransferableTypedData data; + final int sampleRate; + final Duration synthesisTime; + final bool isLast; + + const PocketWorkerAudio({ + required this.generation, + required this.data, + required this.sampleRate, + required this.synthesisTime, + required this.isLast, + }); +} + +class PocketWorkerDone extends PocketWorkerResponse { + final int generation; + + const PocketWorkerDone(this.generation); +} + +enum PocketWorkerFailureKind { load, synthesis, cancelled } + +class PocketWorkerFailure extends PocketWorkerResponse { + final int? generation; + final PocketWorkerFailureKind kind; + final String message; + + const PocketWorkerFailure(this.kind, this.message, {this.generation}); +} + +class PocketWorkerStopped extends PocketWorkerResponse { + const PocketWorkerStopped(); +} + +sealed class _PocketWorkerCommand { + const _PocketWorkerCommand(); +} + +class _Synthesize extends _PocketWorkerCommand { + final int generation; + final String text; + + const _Synthesize(this.generation, this.text); +} + +class _Dispose extends _PocketWorkerCommand { + const _Dispose(); +} + +class PocketVoiceWorker { + final StreamController _responses = + StreamController.broadcast(); + Isolate? _isolate; + SendPort? _commands; + int? _handle; + PocketVoiceFfi? _mainFfi; + ReceivePort? _receive; + Completer? _activeSynthesis; + int? _activeGeneration; + + Stream get responses => _responses.stream; + bool get isReady => _commands != null; + + Future start(String modelPath, {int precision = 0}) async { + if (_isolate != null) return; + final receive = ReceivePort(); + _receive = receive; + final messages = receive.asBroadcastStream(); + _isolate = await Isolate.spawn(_workerMain, ( + receive.sendPort, + modelPath, + precision, + ), debugName: 'buzz-pocket-voice'); + final first = await messages.first; + if (first is! (SendPort, PocketWorkerResponse)) { + throw StateError('Pocket worker sent an invalid startup response.'); + } + _commands = first.$1; + if (first.$2 is PocketWorkerFailure) { + _isolate?.kill(); + _isolate = null; + receive.close(); + _receive = null; + _commands = null; + throw StateError((first.$2 as PocketWorkerFailure).message); + } + final ready = first.$2 as PocketWorkerReady; + _handle = ready.handle; + _mainFfi = PocketVoiceFfi(); + messages.listen((message) { + if (message is! PocketWorkerResponse) return; + if (_finishesActiveSynthesis(message)) { + _activeSynthesis?.complete(); + _activeSynthesis = null; + _activeGeneration = null; + } + _responses.add(message); + }); + } + + void synthesize(int generation, String text) { + final commands = _commands; + final handle = _handle; + final ffi = _mainFfi; + if (commands == null || handle == null || ffi == null) { + throw StateError('Pocket worker is not ready.'); + } + if (_activeSynthesis != null) { + throw StateError('Pocket worker already has an active synthesis.'); + } + // Clear cancellation before publishing the command. Any cancel that + // arrives after this point belongs to this generation and must stay set. + ffi.resetCancel(handle); + _activeGeneration = generation; + _activeSynthesis = Completer(); + commands.send(_Synthesize(generation, text)); + } + + void cancel() { + final handle = _handle; + if (handle != null) _mainFfi?.cancel(handle); + } + + Future cancelAndWait() async { + final active = _activeSynthesis; + if (active == null) return; + cancel(); + await active.future; + } + + Future dispose() async { + if (_responses.isClosed) return; + await cancelAndWait(); + final commands = _commands; + _commands = null; + _handle = null; + _mainFfi = null; + if (commands != null) { + final stopped = _responses.stream.firstWhere( + (response) => response is PocketWorkerStopped, + ); + commands.send(const _Dispose()); + await stopped; + } + _isolate?.kill(); + _isolate = null; + _receive?.close(); + _receive = null; + await _responses.close(); + } + + bool _finishesActiveSynthesis(PocketWorkerResponse response) { + final activeGeneration = _activeGeneration; + if (activeGeneration == null) return false; + return switch (response) { + PocketWorkerAudio(:final generation, :final isLast) => + generation == activeGeneration && isLast, + PocketWorkerDone(:final generation) => generation == activeGeneration, + PocketWorkerFailure(:final generation) => generation == activeGeneration, + _ => false, + }; + } +} + +void _workerMain((SendPort, String, int) startup) { + final output = startup.$1; + final commands = ReceivePort(); + final ffi = PocketVoiceFfi(); + late final int handle; + try { + handle = ffi.create(startup.$2, startup.$3); + } catch (error) { + output.send(( + commands.sendPort, + PocketWorkerFailure(PocketWorkerFailureKind.load, error.toString()), + )); + commands.close(); + return; + } + output.send((commands.sendPort, PocketWorkerReady(handle))); + commands.listen((command) { + switch (command) { + case _Synthesize(): + try { + final chunks = ffi.prepareChunks(handle, command.text); + if (chunks.isEmpty) { + output.send(PocketWorkerDone(command.generation)); + return; + } + for (var index = 0; index < chunks.length; index += 1) { + final stopwatch = Stopwatch()..start(); + final pcm = ffi.synthesize(handle, chunks[index]); + stopwatch.stop(); + output.send( + PocketWorkerAudio( + generation: command.generation, + data: TransferableTypedData.fromList([pcm.bytes]), + sampleRate: pcm.sampleRate, + synthesisTime: stopwatch.elapsed, + isLast: index == chunks.length - 1, + ), + ); + } + } catch (error) { + final message = error.toString(); + output.send( + PocketWorkerFailure( + message.contains('synthesis cancelled') + ? PocketWorkerFailureKind.cancelled + : PocketWorkerFailureKind.synthesis, + message, + generation: command.generation, + ), + ); + } + case _Dispose(): + ffi.destroy(handle); + commands.close(); + output.send(const PocketWorkerStopped()); + Isolate.exit(); + } + }); +} diff --git a/mobile/lib/shared/voice/voice_audio_output.dart b/mobile/lib/shared/voice/voice_audio_output.dart new file mode 100644 index 0000000000..2b65d680ba --- /dev/null +++ b/mobile/lib/shared/voice/voice_audio_output.dart @@ -0,0 +1,48 @@ +import 'dart:async'; + +import 'package:flutter/services.dart'; + +enum VoiceAudioEvent { + started, + completed, + interrupted, + routeLost, + backgrounded, + error, +} + +abstract interface class VoiceAudioOutput { + Stream get events; + Future play(Uint8List pcm, int sampleRate); + Future stop(); +} + +class PlatformVoiceAudioOutput implements VoiceAudioOutput { + static const _channel = MethodChannel('buzz/voice_audio'); + final StreamController _events = + StreamController.broadcast(); + + PlatformVoiceAudioOutput() { + _channel.setMethodCallHandler((call) async { + final event = switch (call.method) { + 'started' => VoiceAudioEvent.started, + 'completed' => VoiceAudioEvent.completed, + 'interrupted' => VoiceAudioEvent.interrupted, + 'routeLost' => VoiceAudioEvent.routeLost, + 'backgrounded' => VoiceAudioEvent.backgrounded, + _ => VoiceAudioEvent.error, + }; + _events.add(event); + }); + } + + @override + Stream get events => _events.stream; + + @override + Future play(Uint8List pcm, int sampleRate) => + _channel.invokeMethod('play', {'pcm': pcm, 'sampleRate': sampleRate}); + + @override + Future stop() => _channel.invokeMethod('stop'); +} diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 55185fc9c6..f8ea7973f2 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -346,7 +346,7 @@ packages: source: hosted version: "1.3.3" ffi: - dependency: transitive + dependency: "direct main" description: name: ffi sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" @@ -446,6 +446,11 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_driver: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" flutter_hooks: dependency: "direct main" description: @@ -568,6 +573,11 @@ packages: url: "https://pub.dev" source: hosted version: "4.0.0" + fuchsia_remote_debug_protocol: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" glob: dependency: transitive description: @@ -712,6 +722,11 @@ packages: url: "https://pub.dev" source: hosted version: "0.2.2" + integration_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" intl: dependency: "direct main" description: @@ -1032,6 +1047,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.5.2" + process: + dependency: transitive + description: + name: process + sha256: c6248e4526673988586e8c00bb22a49210c258dc91df5227d5da9748ecf79744 + url: "https://pub.dev" + source: hosted + version: "5.0.5" provider: dependency: transitive description: @@ -1261,6 +1284,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.1" + sync_http: + dependency: transitive + description: + name: sync_http + sha256: "7f0cd72eca000d2e026bcd6f990b81d0ca06022ef4e32fb257b30d3d1014a961" + url: "https://pub.dev" + source: hosted + version: "0.3.1" term_glyph: dependency: transitive description: @@ -1493,6 +1524,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.3" + webdriver: + dependency: transitive + description: + name: webdriver + sha256: "2f3a14ca026957870cfd9c635b83507e0e51d8091568e90129fbf805aba7cade" + url: "https://pub.dev" + source: hosted + version: "3.1.0" webkit_inspection_protocol: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index b90f50c25a..7d527a8b6a 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -9,6 +9,7 @@ environment: dependencies: flutter: sdk: flutter + ffi: ^2.1.4 hooks_riverpod: ^3.0.3 flutter_hooks: ^0.21.3 lucide_icons_flutter: ^3.1.0 @@ -40,6 +41,8 @@ dependencies: dev_dependencies: flutter_test: sdk: flutter + integration_test: + sdk: flutter flutter_lints: ^6.0.0 custom_lint: ^0.8.0 riverpod_lint: ^3.1.0 diff --git a/mobile/test/features/channels/channel_detail_page_test.dart b/mobile/test/features/channels/channel_detail_page_test.dart index af39eb103a..82f226e52a 100644 --- a/mobile/test/features/channels/channel_detail_page_test.dart +++ b/mobile/test/features/channels/channel_detail_page_test.dart @@ -482,6 +482,8 @@ void main() { expect(find.text('Forum threads are not on mobile yet'), findsNothing); // The compose bar for stream messages should not appear. expect(find.text('Message…'), findsNothing); + // Forum posts are not part of the channel-level voice conversation. + expect(find.byTooltip('Start Pocket voice'), findsNothing); }); testWidgets('renders video attachments from imeta tags in the timeline', ( diff --git a/mobile/test/features/channels/pocket_voice_conversation_test.dart b/mobile/test/features/channels/pocket_voice_conversation_test.dart new file mode 100644 index 0000000000..8c5fa8610f --- /dev/null +++ b/mobile/test/features/channels/pocket_voice_conversation_test.dart @@ -0,0 +1,147 @@ +import 'package:buzz/features/channels/channel_management_provider.dart'; +import 'package:buzz/features/channels/pocket_voice_conversation.dart'; +import 'package:buzz/shared/relay/relay.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + final bot = ChannelMember( + pubkey: 'bot', + role: 'bot', + joinedAt: DateTime.utc(2026), + ); + final person = ChannelMember( + pubkey: 'person', + role: 'member', + joinedAt: DateTime.utc(2026), + ); + + test('baselines history, then selects only new authoritative bot text', () { + final conversation = PocketVoiceConversation(); + expect( + conversation.update( + events: [_event('history', 'bot', 'old')], + members: [bot, person], + currentPubkey: 'self', + ), + isEmpty, + ); + + final spoken = conversation.update( + events: [ + _event('history', 'bot', 'old'), + _event('person', 'person', 'hello'), + _event('self', 'self', 'steer'), + _event('system', 'bot', '[System] working'), + _event('bot', 'bot', 'Assistant answer'), + ], + members: [bot, person], + currentPubkey: 'self', + ); + + expect(spoken.map((event) => event.id), ['bot']); + }); + + test('fails closed while membership is unresolved', () { + final conversation = PocketVoiceConversation(); + conversation.update(events: const [], members: null, currentPubkey: 'self'); + expect( + conversation.update( + events: [_event('bot', 'bot', 'answer')], + members: null, + currentPubkey: 'self', + ), + isEmpty, + ); + expect( + conversation.update( + events: [_event('bot', 'bot', 'answer')], + members: [bot], + currentPubkey: 'self', + ), + isEmpty, + ); + }); + + test('speaks new stream-message v2 bot replies', () { + final conversation = PocketVoiceConversation(); + conversation.update( + events: const [], + members: [bot], + currentPubkey: 'self', + ); + + final spoken = conversation.update( + events: [ + _event( + 'v2', + 'bot', + 'Assistant answer', + kind: EventKind.streamMessageV2, + ), + ], + members: [bot], + currentPubkey: 'self', + ); + + expect(spoken.map((event) => event.id), ['v2']); + }); + + test('thread conversation accepts only direct replies to its head', () { + final conversation = PocketVoiceConversation(); + conversation.update( + events: const [], + members: [bot], + currentPubkey: 'self', + threadHeadId: 'root', + ); + + final spoken = conversation.update( + events: [ + _event('top', 'bot', 'top level'), + _event('direct', 'bot', 'direct', parent: 'root'), + _event('nested', 'bot', 'nested', parent: 'other'), + ], + members: [bot], + currentPubkey: 'self', + threadHeadId: 'root', + ); + + expect(spoken.map((event) => event.id), ['direct']); + }); + + test('never re-speaks retained history after large conversation updates', () { + final conversation = PocketVoiceConversation(); + final history = [ + for (var index = 0; index < 1100; index += 1) + _event('history-$index', 'bot', 'Historical response $index'), + ]; + conversation.update(events: history, members: [bot], currentPubkey: 'self'); + + final spoken = conversation.update( + events: [...history, _event('new', 'bot', 'New response')], + members: [bot], + currentPubkey: 'self', + ); + + expect(spoken.map((event) => event.id), ['new']); + }); +} + +NostrEvent _event( + String id, + String pubkey, + String content, { + String? parent, + int kind = EventKind.streamMessage, +}) => NostrEvent( + id: id, + pubkey: pubkey, + createdAt: 1, + kind: kind, + tags: [ + const ['h', 'channel'], + if (parent != null) ['e', parent, '', 'reply'], + ], + content: content, + sig: '', +); diff --git a/mobile/test/features/channels/send_message_provider_test.dart b/mobile/test/features/channels/send_message_provider_test.dart index f91ce87b2b..951f004de7 100644 --- a/mobile/test/features/channels/send_message_provider_test.dart +++ b/mobile/test/features/channels/send_message_provider_test.dart @@ -66,6 +66,30 @@ void main() { expect(completedIds, isEmpty); expect(removedIds, [localMessages.single.id]); }); + + test('interrupts voice before publishing submitted user text', () async { + final session = _PendingPublishRelaySession(); + var interrupted = false; + final send = SendMessage( + signedEventRelay: SignedEventRelay( + session: session, + nsec: nostr.Keys.generate().nsec, + ), + fetchMembers: (_) async => const [], + readUserCache: () => const {}, + addLocalMessage: (_, _) { + expect(interrupted, isTrue); + }, + completeLocalMessage: (_, _) {}, + removeLocalMessage: (_, _) {}, + onBeforeSend: () async => interrupted = true, + ); + + final result = send(channelId: _channelId, content: 'steer'); + await session.published; + session.accept(); + await result; + }); } const _channelId = '11111111-1111-4111-8111-111111111111'; diff --git a/mobile/test/features/settings/goldens/pocket_voice_settings.png b/mobile/test/features/settings/goldens/pocket_voice_settings.png new file mode 100644 index 0000000000..c104857355 Binary files /dev/null and b/mobile/test/features/settings/goldens/pocket_voice_settings.png differ diff --git a/mobile/test/features/settings/pocket_voice_settings_test.dart b/mobile/test/features/settings/pocket_voice_settings_test.dart new file mode 100644 index 0000000000..2ed3f36c68 --- /dev/null +++ b/mobile/test/features/settings/pocket_voice_settings_test.dart @@ -0,0 +1,82 @@ +import 'dart:io'; + +import 'package:buzz/features/settings/settings_page.dart'; +import 'package:buzz/shared/theme/theme.dart'; +import 'package:buzz/shared/voice/pocket_model_provider.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:package_info_plus/package_info_plus.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +import '../../helpers/widget_helpers.dart'; + +void main() { + testWidgets('shows post-install Pocket model download details', ( + tester, + ) async { + SharedPreferences.setMockInitialValues({}); + PackageInfo.setMockInitialValues( + appName: 'Buzz', + packageName: 'xyz.block.buzz.mobile', + version: '1.0.0', + buildNumber: '1', + buildSignature: '', + ); + final preferences = await SharedPreferences.getInstance(); + await (FontLoader( + 'Inter', + )..addFont(rootBundle.load('assets/fonts/InterVariable.ttf'))).load(); + await (FontLoader('packages/lucide_icons_flutter/Lucide')..addFont( + rootBundle.load('packages/lucide_icons_flutter/assets/lucide.ttf'), + )) + .load(); + const screenshotKey = Key('pocket-voice-settings-screenshot'); + await tester.binding.setSurfaceSize(const Size(430, 932)); + addTearDown(() => tester.binding.setSurfaceSize(null)); + + await tester.pumpWidget( + WidgetHelpers.testable( + overrides: [ + savedPrefsProvider.overrideWithValue(preferences), + pocketModelProvider.overrideWith(_AbsentPocketModelNotifier.new), + ], + child: const RepaintBoundary( + key: screenshotKey, + child: SettingsPage(profileHeader: SizedBox.shrink()), + ), + ), + ); + await tester.pump(); + await tester.pump(const Duration(milliseconds: 500)); + + expect(find.text('Pocket voice'), findsOneWidget); + expect(find.text('Model size'), findsOneWidget); + expect(find.text('Higher quality'), findsOneWidget); + expect(find.text('Not downloaded'), findsOneWidget); + expect(find.textContaining('Downloads 440 MB'), findsOneWidget); + expect(find.text('440 MB'), findsOneWidget); + + // Flutter's text rasterization differs between macOS and the Linux CI + // runner even with the same bundled fonts. Keep the pixel baseline on the + // platform that owns the pixel baseline; the assertions above + // remain the cross-platform contract for the download state. + if (Platform.isMacOS) { + await expectLater( + find.byKey(screenshotKey), + matchesGoldenFile('goldens/pocket_voice_settings.png'), + ); + } + + await tester.tap(find.text('Model size')); + await tester.pumpAndSettle(); + expect(find.text('Faster'), findsOneWidget); + expect(find.text('Hybrid INT8 · 165 MB'), findsOneWidget); + }); +} + +class _AbsentPocketModelNotifier extends PocketModelNotifier { + @override + PocketModelState build() => + const PocketModelState(phase: PocketModelPhase.absent); +} diff --git a/mobile/test/shared/voice/pocket_model_downloader_test.dart b/mobile/test/shared/voice/pocket_model_downloader_test.dart new file mode 100644 index 0000000000..bdbf1f1735 --- /dev/null +++ b/mobile/test/shared/voice/pocket_model_downloader_test.dart @@ -0,0 +1,428 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:buzz/shared/voice/pocket_model_downloader.dart'; +import 'package:buzz/shared/voice/pocket_model_manifest.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; + +void main() { + late Directory temporary; + late PocketModelArtifact artifact; + + setUp(() async { + temporary = await Directory.systemTemp.createTemp('buzz-pocket-model-'); + artifact = PocketModelArtifact( + name: 'tiny.bin', + url: Uri.parse('https://example.test/tiny.bin'), + size: 5, + sha256: + '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e730' + '43362938b9824', + ); + }); + + tearDown(() async { + if (await temporary.exists()) await temporary.delete(recursive: true); + }); + + test( + 'installs verified files atomically and reports exact progress', + () async { + final excluded = []; + final progress = <(int, int, String)>[]; + final downloader = _downloader( + temporary, + artifact, + client: MockClient((_) async => http.Response('hello', 200)), + excludeFromBackup: excluded.add, + ); + final legacy = Directory('${temporary.path}/buzz/models/pocket-tts/v3'); + await legacy.create(recursive: true); + await File('${legacy.path}/sentinel').writeAsString('January'); + + final directory = await downloader.install( + (downloaded, total, file) => progress.add((downloaded, total, file)), + ); + + expect(await downloader.verify(directory, hashContents: true), isTrue); + expect(await File('${directory.path}/tiny.bin').readAsString(), 'hello'); + final manifest = + jsonDecode( + await File( + '${directory.path}/.buzz-model-manifest', + ).readAsString(), + ) + as Map; + expect(manifest['version'], pocketModelVersion); + expect(manifest['revision'], pocketModelRevision); + expect(manifest['precision'], PocketModelVariant.higherQuality.precision); + expect(progress.last, (5, 5, 'tiny.bin')); + expect(await legacy.exists(), isTrue); + expect(excluded, hasLength(2)); + expect( + excluded.first, + contains('.pocket-tts-v$pocketModelVersion-fp32.install-'), + ); + expect(excluded.last, directory.path); + expect( + await directory.parent + .list() + .where((entry) => entry.path.contains('.install-')) + .isEmpty, + isTrue, + ); + }, + ); + + test('rejects a checksum mismatch without replacing an install', () async { + final legacy = Directory('${temporary.path}/buzz/models/pocket-tts/v3'); + await legacy.create(recursive: true); + await File('${legacy.path}/sentinel').writeAsString('January'); + final downloader = _downloader( + temporary, + artifact, + client: MockClient((_) async => http.Response('wrong', 200)), + ); + + await expectLater( + downloader.install((_, _, _) {}), + throwsA(isA()), + ); + + expect( + await downloader.modelDirectory().then((dir) => dir.exists()), + isFalse, + ); + expect(await File('${legacy.path}/sentinel').readAsString(), 'January'); + }); + + test('fails before download when the model and reserve do not fit', () async { + final downloader = _downloader( + temporary, + artifact, + client: MockClient((_) async => http.Response('hello', 200)), + capacity: 4, + ); + + await expectLater( + downloader.install((_, _, _) {}), + throwsA( + isA() + .having((error) => error.available, 'available', 4) + .having((error) => error.required, 'required', greaterThan(5)), + ), + ); + }); + + test('cancellation closes the transfer and removes staging files', () async { + final client = _PendingClient(); + final downloader = _downloader(temporary, artifact, client: client); + + final install = downloader.install((_, _, _) {}); + await client.started.future; + downloader.cancel(); + + await expectLater(install, throwsA(isA())); + expect(client.closed, isTrue); + expect( + await downloader.modelDirectory().then((directory) => directory.exists()), + isFalse, + ); + }); + + test('recovers an old install left by an interrupted atomic swap', () async { + final parent = Directory('${temporary.path}/buzz/models/pocket-tts'); + final backup = Directory( + '${parent.path}/.pocket-tts-v$pocketModelVersion-fp32.old-crash', + ); + await backup.create(recursive: true); + await File('${backup.path}/sentinel').writeAsString('previous install'); + final client = _PendingClient(); + final downloader = _downloader(temporary, artifact, client: client); + + final install = downloader.install((_, _, _) {}); + await client.started.future; + final finalDirectory = await downloader.modelDirectory(); + expect( + await File('${finalDirectory.path}/sentinel').readAsString(), + 'previous install', + ); + downloader.cancel(); + await expectLater(install, throwsA(isA())); + }); + + test( + 'migrates the verified v4 FP32 cache into its v5 variant path', + () async { + final downloader = _downloader( + temporary, + artifact, + client: MockClient((_) async => http.Response('unused', 500)), + ); + final root = Directory('${temporary.path}/buzz/models/pocket-tts'); + final legacy = Directory('${root.path}/v$pocketLegacyFp32Version'); + await legacy.create(recursive: true); + await File('${legacy.path}/tiny.bin').writeAsString('hello'); + await File( + '${legacy.path}/MODEL_LICENSE.txt', + ).writeAsString(pocketModelLicenseText); + await File('${legacy.path}/.buzz-model-manifest').writeAsString( + jsonEncode({ + 'version': pocketLegacyFp32Version, + 'revision': pocketModelRevision, + 'precision': PocketModelVariant.higherQuality.precision, + 'complete': true, + }), + ); + + expect(await downloader.isReady(), isTrue); + + final migrated = await downloader.modelDirectory(); + expect(await File('${migrated.path}/tiny.bin').readAsString(), 'hello'); + expect(await legacy.exists(), isFalse); + final manifest = + jsonDecode( + await File( + '${migrated.path}/.buzz-model-manifest', + ).readAsString(), + ) + as Map; + expect(manifest['version'], pocketModelVersion); + expect(manifest['precision'], 'fp32'); + }, + ); + + test('recovers and migrates a verified v4 atomic-swap backup', () async { + final downloader = _downloader( + temporary, + artifact, + client: MockClient((_) async => http.Response('unused', 500)), + ); + final root = Directory('${temporary.path}/buzz/models/pocket-tts'); + final backup = Directory('${root.path}/.pocket-tts-v4.old-crash'); + await backup.create(recursive: true); + await File('${backup.path}/tiny.bin').writeAsString('hello'); + await File( + '${backup.path}/MODEL_LICENSE.txt', + ).writeAsString(pocketModelLicenseText); + await File('${backup.path}/.buzz-model-manifest').writeAsString( + jsonEncode({ + 'version': pocketLegacyFp32Version, + 'revision': pocketModelRevision, + 'precision': PocketModelVariant.higherQuality.precision, + 'complete': true, + }), + ); + + expect(await downloader.isReady(), isTrue); + + final migrated = await downloader.modelDirectory(); + expect(await File('${migrated.path}/tiny.bin').readAsString(), 'hello'); + expect(await backup.exists(), isFalse); + expect( + await Directory('${root.path}/v$pocketLegacyFp32Version').exists(), + isFalse, + ); + }); + + test( + 'resumes a v5 migration staged after its manifest was rewritten', + () async { + final excluded = []; + final downloader = _downloader( + temporary, + artifact, + client: MockClient((_) async => http.Response('unused', 500)), + excludeFromBackup: excluded.add, + ); + final root = Directory('${temporary.path}/buzz/models/pocket-tts'); + final staging = Directory( + '${root.path}/.pocket-tts-v$pocketModelVersion-fp32.' + 'migrate-previous-process', + ); + await staging.create(recursive: true); + await File('${staging.path}/tiny.bin').writeAsString('hello'); + await File( + '${staging.path}/MODEL_LICENSE.txt', + ).writeAsString(pocketModelLicenseText); + await File('${staging.path}/.buzz-model-manifest').writeAsString( + jsonEncode({ + 'version': pocketModelVersion, + 'revision': pocketModelRevision, + 'precision': PocketModelVariant.higherQuality.precision, + 'complete': true, + }), + ); + + expect(await downloader.isReady(), isTrue); + + final migrated = await downloader.modelDirectory(); + expect(await File('${migrated.path}/tiny.bin').readAsString(), 'hello'); + expect(await staging.exists(), isFalse); + expect( + await Directory('${root.path}/v$pocketLegacyFp32Version').exists(), + isFalse, + ); + expect(excluded, [migrated.path]); + }, + ); + + test('replaces clients and retries when sending never completes', () async { + final clients = <_NeverSendingClient>[]; + final downloader = _downloader( + temporary, + artifact, + clientFactory: () { + final client = _NeverSendingClient(); + clients.add(client); + return client; + }, + sendTimeout: const Duration(milliseconds: 5), + ); + + await expectLater( + downloader.install((_, _, _) {}), + throwsA(isA()), + ); + + expect(clients, hasLength(3)); + expect(clients.every((client) => client.closed), isTrue); + }); + + test('replaces the client and retries after a body idle timeout', () async { + final stalled = _StallingBodyClient(); + final succeeding = MockClient((_) async => http.Response('hello', 200)); + final clients = [stalled, succeeding]; + var nextClient = 0; + final downloader = _downloader( + temporary, + artifact, + clientFactory: () => clients[nextClient++], + bodyIdleTimeout: const Duration(milliseconds: 5), + ); + + final directory = await downloader.install((_, _, _) {}); + + expect(await File('${directory.path}/tiny.bin').readAsString(), 'hello'); + expect(stalled.closed, isTrue); + expect(nextClient, 2); + }); + + test('both April manifests match the frozen shared runtime metadata', () { + expect(pocketModelVersion, '5'); + expect(pocketModelRevision, '58a6d00cf13d239b6748cb0769f35c580a8f606c'); + for (final variant in PocketModelVariant.values) { + final artifacts = pocketModelArtifacts[variant]!; + expect( + artifacts + .where((artifact) => artifact.name != 'reference_sample.wav') + .where((artifact) => artifact.name != 'LICENSE') + .fold(0, (total, artifact) => total + artifact.size), + variant.coreBytes, + ); + expect( + artifacts.fold(0, (total, artifact) => total + artifact.size), + variant.runtimeBytes, + ); + expect( + artifacts.map((artifact) => artifact.name), + containsAll([ + 'bundle.json', + 'bos_before_voice.npy', + 'tokenizer.model', + 'mimi_encoder.onnx', + 'text_conditioner.onnx', + ]), + ); + } + expect( + pocketModelArtifacts[PocketModelVariant.faster]!.map( + (artifact) => artifact.name, + ), + containsAll([ + 'flow_lm_main_int8.onnx', + 'flow_lm_flow_int8.onnx', + 'mimi_decoder_int8.onnx', + ]), + ); + expect( + pocketModelArtifacts[PocketModelVariant.faster]!.map( + (artifact) => artifact.name, + ), + isNot(contains('mimi_encoder_int8.onnx')), + ); + expect(pocketModelLicenseText, contains('April 2026 ONNX export')); + expect(pocketModelLicenseText, contains(pocketModelRevision)); + }); +} + +PocketModelDownloader _downloader( + Directory support, + PocketModelArtifact artifact, { + http.Client? client, + http.Client Function()? clientFactory, + int capacity = 1024 * 1024 * 1024, + void Function(String)? excludeFromBackup, + Duration sendTimeout = const Duration(seconds: 30), + Duration bodyIdleTimeout = const Duration(seconds: 30), +}) => PocketModelDownloader( + clientFactory: clientFactory ?? () => client!, + applicationSupportDirectory: () async => support, + availableCapacity: (_) async => capacity, + excludeFromBackup: (path) async => excludeFromBackup?.call(path), + artifacts: [artifact], + installationId: 'test', + sendTimeout: sendTimeout, + bodyIdleTimeout: bodyIdleTimeout, +); + +class _PendingClient extends http.BaseClient { + final started = Completer(); + final _response = StreamController>(); + bool closed = false; + + @override + Future send(http.BaseRequest request) async { + started.complete(); + return http.StreamedResponse(_response.stream, 200, contentLength: 5); + } + + @override + void close() { + closed = true; + unawaited(_response.close()); + } +} + +class _NeverSendingClient extends http.BaseClient { + bool closed = false; + + @override + Future send(http.BaseRequest request) => + Completer().future; + + @override + void close() { + closed = true; + } +} + +class _StallingBodyClient extends http.BaseClient { + final _response = StreamController>(); + bool closed = false; + + @override + Future send(http.BaseRequest request) async { + scheduleMicrotask(() => _response.add('he'.codeUnits)); + return http.StreamedResponse(_response.stream, 200, contentLength: 5); + } + + @override + void close() { + closed = true; + unawaited(_response.close()); + } +} diff --git a/mobile/test/shared/voice/pocket_model_provider_test.dart b/mobile/test/shared/voice/pocket_model_provider_test.dart new file mode 100644 index 0000000000..32d364a27d --- /dev/null +++ b/mobile/test/shared/voice/pocket_model_provider_test.dart @@ -0,0 +1,222 @@ +import 'dart:io'; + +import 'package:buzz/shared/theme/theme.dart'; +import 'package:buzz/shared/voice/pocket_model_downloader.dart'; +import 'package:buzz/shared/voice/pocket_model_manifest.dart'; +import 'package:buzz/shared/voice/pocket_model_provider.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +void main() { + late Directory temporary; + + setUp(() async { + temporary = await Directory.systemTemp.createTemp('buzz-pocket-selection-'); + }); + + tearDown(() async { + if (await temporary.exists()) await temporary.delete(recursive: true); + }); + + test('persists Faster only after download and engine validation', () async { + SharedPreferences.setMockInitialValues({}); + final preferences = await SharedPreferences.getInstance(); + final fp32 = _FakeDownloader( + variant: PocketModelVariant.higherQuality, + directory: Directory('${temporary.path}/fp32'), + ready: true, + ); + final int8 = _FakeDownloader( + variant: PocketModelVariant.faster, + directory: Directory('${temporary.path}/int8'), + ); + final validated = []; + final container = ProviderContainer( + overrides: [ + savedPrefsProvider.overrideWithValue(preferences), + pocketModelDownloaderProvider.overrideWith( + (_, variant) => + variant == PocketModelVariant.higherQuality ? fp32 : int8, + ), + pocketModelValidatorProvider.overrideWithValue((_, variant) async { + validated.add(variant); + }), + ], + ); + addTearDown(container.dispose); + await _settle(container); + fp32.deleteCount = 0; + int8.deleteCount = 0; + + await container + .read(pocketModelProvider.notifier) + .select(PocketModelVariant.faster); + + final state = container.read(pocketModelProvider); + expect(state.phase, PocketModelPhase.ready); + expect(state.variant, PocketModelVariant.faster); + expect(state.path, int8.directory.path); + expect(int8.installCount, 1); + expect(validated, [PocketModelVariant.faster]); + expect(preferences.getString('pocket_model_precision'), 'int8'); + expect(fp32.deleteCount, 1); + }); + + test('keeps the active cache and preference when validation fails', () async { + SharedPreferences.setMockInitialValues({'pocket_model_precision': 'fp32'}); + final preferences = await SharedPreferences.getInstance(); + final fp32 = _FakeDownloader( + variant: PocketModelVariant.higherQuality, + directory: Directory('${temporary.path}/fp32'), + ready: true, + ); + final int8 = _FakeDownloader( + variant: PocketModelVariant.faster, + directory: Directory('${temporary.path}/int8'), + ); + final container = ProviderContainer( + overrides: [ + savedPrefsProvider.overrideWithValue(preferences), + pocketModelDownloaderProvider.overrideWith( + (_, variant) => + variant == PocketModelVariant.higherQuality ? fp32 : int8, + ), + pocketModelValidatorProvider.overrideWithValue( + (_, _) async => throw StateError('unsupported device'), + ), + ], + ); + addTearDown(container.dispose); + await _settle(container); + fp32.deleteCount = 0; + int8.deleteCount = 0; + + await container + .read(pocketModelProvider.notifier) + .select(PocketModelVariant.faster); + + final state = container.read(pocketModelProvider); + expect(state.phase, PocketModelPhase.ready); + expect(state.variant, PocketModelVariant.higherQuality); + expect(state.path, fp32.directory.path); + expect(state.message, contains('Higher quality was kept')); + expect(preferences.getString('pocket_model_precision'), 'fp32'); + expect(fp32.deleteCount, 0); + expect(int8.deleteCount, 1); + }); + + test( + 'falls back to an installed variant when the preference is unavailable', + () async { + SharedPreferences.setMockInitialValues({ + 'pocket_model_precision': 'int8', + }); + final preferences = await SharedPreferences.getInstance(); + final fp32 = _FakeDownloader( + variant: PocketModelVariant.higherQuality, + directory: Directory('${temporary.path}/fp32'), + ready: true, + ); + final int8 = _FakeDownloader( + variant: PocketModelVariant.faster, + directory: Directory('${temporary.path}/int8'), + ); + final container = ProviderContainer( + overrides: [ + savedPrefsProvider.overrideWithValue(preferences), + pocketModelDownloaderProvider.overrideWith( + (_, variant) => + variant == PocketModelVariant.higherQuality ? fp32 : int8, + ), + ], + ); + addTearDown(container.dispose); + + await _settle(container); + + final state = container.read(pocketModelProvider); + expect(state.variant, PocketModelVariant.higherQuality); + expect(state.message, contains('Faster was unavailable')); + expect(preferences.getString('pocket_model_precision'), 'fp32'); + }, + ); + + test('readiness cleanup removes a stale inactive variant cache', () async { + SharedPreferences.setMockInitialValues({'pocket_model_precision': 'fp32'}); + final preferences = await SharedPreferences.getInstance(); + final fp32 = _FakeDownloader( + variant: PocketModelVariant.higherQuality, + directory: Directory('${temporary.path}/fp32'), + ready: true, + ); + final int8 = _FakeDownloader( + variant: PocketModelVariant.faster, + directory: Directory('${temporary.path}/int8'), + ready: true, + ); + final container = ProviderContainer( + overrides: [ + savedPrefsProvider.overrideWithValue(preferences), + pocketModelDownloaderProvider.overrideWith( + (_, variant) => + variant == PocketModelVariant.higherQuality ? fp32 : int8, + ), + ], + ); + addTearDown(container.dispose); + + await _settle(container); + + expect(container.read(pocketModelProvider).variant, fp32.variant); + expect(int8.deleteCount, 1); + }); +} + +Future _settle(ProviderContainer container) async { + container.read(pocketModelProvider); + for (var attempt = 0; attempt < 20; attempt += 1) { + await Future.delayed(Duration.zero); + if (container.read(pocketModelProvider).phase != + PocketModelPhase.checking) { + return; + } + } + fail('Pocket model provider did not settle.'); +} + +class _FakeDownloader extends PocketModelDownloader { + final Directory directory; + bool ready; + int installCount = 0; + int deleteCount = 0; + + _FakeDownloader({ + required super.variant, + required this.directory, + this.ready = false, + }); + + @override + Future modelDirectory() async => directory; + + @override + Future isReady() async => ready; + + @override + Future install(PocketDownloadProgress onProgress) async { + installCount += 1; + ready = true; + onProgress(variant.runtimeBytes, variant.runtimeBytes, 'complete'); + return directory; + } + + @override + Future deleteCache() async { + deleteCount += 1; + ready = false; + } + + @override + Future deleteLegacyCaches() async {} +} diff --git a/mobile/test/shared/voice/pocket_voice_controller_test.dart b/mobile/test/shared/voice/pocket_voice_controller_test.dart new file mode 100644 index 0000000000..d481a56b0f --- /dev/null +++ b/mobile/test/shared/voice/pocket_voice_controller_test.dart @@ -0,0 +1,421 @@ +import 'dart:async'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:buzz/shared/relay/relay.dart'; +import 'package:buzz/shared/voice/pocket_model_provider.dart'; +import 'package:buzz/shared/voice/pocket_model_manifest.dart'; +import 'package:buzz/shared/voice/pocket_voice_controller.dart'; +import 'package:buzz/shared/voice/pocket_voice_worker.dart'; +import 'package:buzz/shared/voice/voice_audio_output.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +void main() { + test('queues assistant messages and plays every chunk in order', () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + await notifier.enable('conversation'); + notifier.speak('conversation', 'First response.'); + notifier.speak('conversation', 'Second response.'); + + expect(worker.syntheses, [(1, 'First response.')]); + + worker.emitAudio(1, [1, 2], isLast: false); + await _flush(); + expect(output.played, hasLength(1)); + expect(output.played[0].$1, [1, 2]); + expect(output.played[0].$2, 24000); + expect( + container.read(pocketVoiceProvider).phase, + PocketVoicePhase.synthesizing, + ); + output.started(); + await _flush(); + expect( + container.read(pocketVoiceProvider).phase, + PocketVoicePhase.speaking, + ); + + worker.emitAudio(1, [3, 4], isLast: true); + await _flush(); + expect(output.played, hasLength(1)); + output.complete(); + await _flush(); + expect(output.played, hasLength(2)); + expect(output.played[1].$1, [3, 4]); + expect(worker.syntheses, [(1, 'First response.')]); + + output.complete(); + await _flush(); + expect(worker.syntheses, [(1, 'First response.'), (2, 'Second response.')]); + + worker.emitAudio(2, [5, 6], isLast: true); + await _flush(); + output.complete(); + await _flush(); + + expect(output.played, hasLength(3)); + expect(output.played[0].$1, [1, 2]); + expect(output.played[1].$1, [3, 4]); + expect(output.played[2].$1, [5, 6]); + expect( + container.read(pocketVoiceProvider).phase, + PocketVoicePhase.listening, + ); + }); + + test( + 'disable wins an overlapping engine startup and retains warm worker', + () async { + final worker = _FakeWorker(startPaused: true); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + final enabling = notifier.enable('conversation'); + await _flush(); + final disabling = notifier.disable(); + worker.finishStart(); + await Future.wait([enabling, disabling]); + + expect(container.read(pocketVoiceProvider).phase, PocketVoicePhase.off); + expect(worker.startCount, 1); + expect(worker.disposeCount, 0); + + await notifier.enable('next-conversation'); + expect(worker.startCount, 1); + expect( + container.read(pocketVoiceProvider).phase, + PocketVoicePhase.listening, + ); + }, + ); + + test('overlapping enables share an in-flight engine startup', () async { + final worker = _FakeWorker(startPaused: true); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + final first = notifier.enable('first-conversation'); + await _flush(); + final second = notifier.enable('second-conversation'); + await _flush(); + + expect(worker.startCount, 1); + expect(worker.disposeCount, 0); + + worker.finishStart(); + await Future.wait([first, second]); + + expect(worker.startCount, 1); + expect(worker.disposeCount, 0); + expect( + container.read(pocketVoiceProvider).conversationKey, + 'second-conversation', + ); + expect( + container.read(pocketVoiceProvider).phase, + PocketVoicePhase.listening, + ); + }); + + test('model selection releases the resident engine', () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + await notifier.enable('conversation'); + await notifier.releaseEngineForModelSelection(); + + expect(container.read(pocketVoiceProvider).phase, PocketVoicePhase.off); + expect(worker.disposeCount, 1); + }); + + test( + 'model selection waits for an in-flight engine before release', + () async { + final worker = _FakeWorker(startPaused: true); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + final enabling = notifier.enable('conversation'); + await _flush(); + final releasing = notifier.releaseEngineForModelSelection(); + await _flush(); + expect(worker.disposeCount, 0); + + worker.finishStart(); + await Future.wait([enabling, releasing]); + + expect(container.read(pocketVoiceProvider).phase, PocketVoicePhase.off); + expect(worker.disposeCount, 1); + }, + ); + + test( + 'preserves text submitted while the resident engine is loading', + () async { + final worker = _FakeWorker(startPaused: true); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + final longResponse = 'A' * 2100; + + final enabling = notifier.enable('conversation'); + await _flush(); + notifier.speak('conversation', longResponse); + worker.finishStart(); + await enabling; + + expect(worker.syntheses, [(1, longResponse)]); + }, + ); + + test( + 'queues new responses during interrupt and rejects them during disable', + () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + await notifier.enable('conversation'); + notifier.speak('conversation', 'Before interrupt.'); + worker.pauseCancellation(); + final interrupting = notifier.interrupt(); + await _flush(); + notifier.speak('conversation', 'After steering.'); + expect(worker.syntheses, [(1, 'Before interrupt.')]); + + worker.finishCancellation(); + await interrupting; + expect(worker.syntheses, [ + (1, 'Before interrupt.'), + (2, 'After steering.'), + ]); + + worker.pauseCancellation(); + final disabling = notifier.disable(); + notifier.speak('conversation', 'Must not be spoken.'); + worker.finishCancellation(); + await disabling; + expect(worker.syntheses, hasLength(2)); + expect(container.read(pocketVoiceProvider).phase, PocketVoicePhase.off); + }, + ); + + test('surfaces asynchronous native playback errors', () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + await notifier.enable('conversation'); + notifier.speak('conversation', 'Response.'); + worker.emitAudio(1, [1, 2], isLast: true); + await _flush(); + output.fail(); + await _flush(); + + final state = container.read(pocketVoiceProvider); + expect(state.phase, PocketVoicePhase.error); + expect(state.failureKind, PocketVoiceFailureKind.playback); + expect(state.error, 'Pocket voice playback failed.'); + expect(worker.cancelCount, greaterThan(0)); + + notifier.speak('conversation', 'Must wait for explicit recovery.'); + await _flush(); + expect(worker.syntheses, [(1, 'Response.')]); + expect(container.read(pocketVoiceProvider), same(state)); + }); + + test('classifies synthesis failures for platform fallback routing', () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = _container(worker, output); + addTearDown(container.dispose); + final notifier = container.read(pocketVoiceProvider.notifier); + + await notifier.enable('conversation'); + notifier.speak('conversation', 'Response.'); + worker.emitFailure( + 1, + PocketWorkerFailureKind.synthesis, + 'Pocket synthesis failed.', + ); + await _flush(); + + final state = container.read(pocketVoiceProvider); + expect(state.phase, PocketVoicePhase.error); + expect(state.failureKind, PocketVoiceFailureKind.synthesis); + expect(state.error, 'Pocket synthesis failed.'); + }); + + test('loads Faster through the existing worker precision seam', () async { + final worker = _FakeWorker(); + final output = _FakeAudioOutput(); + final container = ProviderContainer( + overrides: [ + relayConfigProvider.overrideWith(_TestRelayConfigNotifier.new), + pocketModelProvider.overrideWith(_ReadyFasterModelNotifier.new), + pocketVoiceWorkerFactoryProvider.overrideWithValue(() => worker), + voiceAudioOutputProvider.overrideWithValue(output), + ], + ); + addTearDown(container.dispose); + + await container.read(pocketVoiceProvider.notifier).enable('conversation'); + + expect(worker.startedPrecision, PocketModelVariant.faster.nativePrecision); + }); +} + +ProviderContainer _container(_FakeWorker worker, _FakeAudioOutput output) => + ProviderContainer( + overrides: [ + relayConfigProvider.overrideWith(_TestRelayConfigNotifier.new), + pocketModelProvider.overrideWith(_ReadyPocketModelNotifier.new), + pocketVoiceWorkerFactoryProvider.overrideWithValue(() => worker), + voiceAudioOutputProvider.overrideWithValue(output), + ], + ); + +Future _flush() => Future.delayed(Duration.zero); + +class _ReadyPocketModelNotifier extends PocketModelNotifier { + @override + PocketModelState build() => const PocketModelState( + phase: PocketModelPhase.ready, + path: '/tmp/pocket-model', + ); +} + +class _ReadyFasterModelNotifier extends PocketModelNotifier { + @override + PocketModelState build() => const PocketModelState( + phase: PocketModelPhase.ready, + variant: PocketModelVariant.faster, + path: '/tmp/pocket-model-int8', + ); +} + +class _TestRelayConfigNotifier extends RelayConfigNotifier { + @override + RelayConfig build() => const RelayConfig(baseUrl: 'http://localhost:3000'); +} + +class _FakeWorker extends PocketVoiceWorker { + final StreamController _controller = + StreamController.broadcast(); + final Completer? _startGate; + final List<(int, String)> syntheses = []; + Completer? _cancelGate; + bool _ready = false; + int startCount = 0; + int disposeCount = 0; + int cancelCount = 0; + int? startedPrecision; + + _FakeWorker({bool startPaused = false}) + : _startGate = startPaused ? Completer() : null; + + @override + Stream get responses => _controller.stream; + + @override + bool get isReady => _ready; + + @override + Future start(String modelPath, {int precision = 0}) async { + startCount += 1; + startedPrecision = precision; + await _startGate?.future; + _ready = true; + } + + void finishStart() => _startGate?.complete(); + + @override + void synthesize(int generation, String text) { + syntheses.add((generation, text)); + } + + void emitAudio(int generation, List bytes, {required bool isLast}) { + _controller.add( + PocketWorkerAudio( + generation: generation, + data: TransferableTypedData.fromList([Uint8List.fromList(bytes)]), + sampleRate: 24000, + synthesisTime: const Duration(milliseconds: 1), + isLast: isLast, + ), + ); + } + + void emitFailure( + int generation, + PocketWorkerFailureKind kind, + String message, + ) { + _controller.add(PocketWorkerFailure(kind, message, generation: generation)); + } + + @override + void cancel() { + cancelCount += 1; + } + + @override + Future cancelAndWait() async { + await _cancelGate?.future; + _cancelGate = null; + } + + void pauseCancellation() => _cancelGate = Completer(); + + void finishCancellation() => _cancelGate?.complete(); + + @override + Future dispose() async { + disposeCount += 1; + await _controller.close(); + } +} + +class _FakeAudioOutput implements VoiceAudioOutput { + final StreamController _controller = + StreamController.broadcast(); + final List<(List, int)> played = []; + + @override + Stream get events => _controller.stream; + + @override + Future play(Uint8List pcm, int sampleRate) async { + played.add((pcm.toList(), sampleRate)); + } + + void complete() => _controller.add(VoiceAudioEvent.completed); + + void started() => _controller.add(VoiceAudioEvent.started); + + void fail() => _controller.add(VoiceAudioEvent.error); + + @override + Future stop() async {} +} diff --git a/scripts/mobile-voice-native.sh b/scripts/mobile-voice-native.sh new file mode 100755 index 0000000000..17ba2cb417 --- /dev/null +++ b/scripts/mobile-voice-native.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +version="1.13.4" +release_url="https://github.com/k2-fsa/sherpa-onnx/releases/download/v${version}" +cache_root="${BUZZ_VOICE_NATIVE_CACHE:-${HOME}/Library/Caches/buzz-mobile-voice/sherpa-onnx-v${version}}" +generated_root="${repo_root}/mobile/.generated/voice" + +android_archive="sherpa-onnx-v${version}-android.tar.bz2" +android_sha256="7983fc3de23f6e64148f2fb05fa94a2efaa8c0516cc1573383dc5c7d4d2a43b0" +ios_archive="sherpa-onnx-v${version}-ios.tar.bz2" +ios_sha256="596f33bff80046a52144745745fe54d55e8b23659d92209f5ab7d94c1259fe6d" + +usage() { + echo "usage: $0 fetch|build-ios|build-ios-current|build-android|build-all" >&2 + exit 2 +} + +verify_sha256() { + local path="$1" + local expected="$2" + local actual + actual="$(shasum -a 256 "$path" | awk '{print $1}')" + if [[ "$actual" != "$expected" ]]; then + echo "checksum mismatch for $path: expected $expected, got $actual" >&2 + return 1 + fi +} + +fetch_archive() { + local archive="$1" + local expected="$2" + mkdir -p "$cache_root" + if [[ -f "${cache_root}/${archive}" ]] && + ! verify_sha256 "${cache_root}/${archive}" "$expected"; then + rm "${cache_root:?}/${archive}" + fi + if [[ ! -f "${cache_root}/${archive}" ]]; then + curl --fail --location --retry 3 \ + --output "${cache_root}/${archive}.part" \ + "${release_url}/${archive}" + verify_sha256 "${cache_root}/${archive}.part" "$expected" + mv "${cache_root}/${archive}.part" "${cache_root}/${archive}" + fi +} + +extract_archive() { + local archive="$1" + local expected="$2" + local destination="$3" + fetch_archive "$archive" "$expected" + if [[ -f "${destination}/.complete" ]]; then + return + fi + local staging="${destination}.install-$$" + rm -rf "$staging" + mkdir -p "$staging" + tar xjf "${cache_root}/${archive}" -C "$staging" + touch "${staging}/.complete" + rm -rf "$destination" + mv "$staging" "$destination" +} + +fetch_native() { + extract_archive "$android_archive" "$android_sha256" "${cache_root}/android" + extract_archive "$ios_archive" "$ios_sha256" "${cache_root}/ios" +} + +ios_lib_dir() { + local sdk="$1" + local arch="$2" + local sherpa_slice + local ort_slice + if [[ "$sdk" == "iphoneos" ]]; then + sherpa_slice="ios-arm64" + ort_slice="ios-arm64" + else + sherpa_slice="ios-arm64_x86_64-simulator" + ort_slice="ios-arm64_x86_64-simulator" + fi + local destination="${generated_root}/ios/native/${sdk}-${arch}" + mkdir -p "$destination" + local sherpa_binary="${cache_root}/ios/build-ios/sherpa-onnx.xcframework/${sherpa_slice}/libsherpa-onnx.a" + if [[ "$sdk" == "iphoneos" ]]; then + cp "$sherpa_binary" "${destination}/libsherpa-onnx.a" + else + lipo "$sherpa_binary" -thin "$arch" -output "${destination}/libsherpa-onnx.a" + fi + local ort_binary="${cache_root}/ios/build-ios/ios-onnxruntime/1.27.0/onnxruntime.xcframework/${ort_slice}/onnxruntime.framework/onnxruntime" + lipo "$ort_binary" -thin "$arch" -output "${destination}/libonnxruntime.a" + printf '%s\n' "$destination" +} + +build_ios_target() { + local target="$1" + local sdk="$2" + local arch="$3" + local lib_dir + lib_dir="$(ios_lib_dir "$sdk" "$arch")" + IPHONEOS_DEPLOYMENT_TARGET=16.0 SHERPA_ONNX_LIB_DIR="$lib_dir" \ + cargo build --release -p buzz-voice-mobile --target "$target" + local destination="${generated_root}/ios/${sdk}-${arch}" + mkdir -p "$destination" + rm -f "$destination/libbuzz_voice_mobile.a" + cp "${repo_root}/target/${target}/release/libbuzz_voice_mobile.a" "$destination/" +} + +build_ios() { + fetch_native + build_ios_target "aarch64-apple-ios" "iphoneos" "arm64" + build_ios_target "aarch64-apple-ios-sim" "iphonesimulator" "arm64" + build_ios_target "x86_64-apple-ios" "iphonesimulator" "x86_64" +} + +build_ios_current() { + fetch_native + local platform="${PLATFORM_NAME:-${SDK_NAME%%[0-9]*}}" + local arch="${CURRENT_ARCH:-}" + if [[ -z "$arch" || "$arch" == "undefined_arch" ]]; then + if [[ " ${ARCHS:-} " == *" arm64 "* ]]; then + arch="arm64" + elif [[ " ${ARCHS:-} " == *" x86_64 "* ]]; then + arch="x86_64" + fi + fi + case "${platform}:${arch}" in + iphoneos:arm64) + build_ios_target "aarch64-apple-ios" "iphoneos" "arm64" + ;; + iphonesimulator:arm64) + build_ios_target "aarch64-apple-ios-sim" "iphonesimulator" "arm64" + ;; + iphonesimulator:x86_64) + build_ios_target "x86_64-apple-ios" "iphonesimulator" "x86_64" + ;; + *) + echo "unsupported iOS platform/architecture: ${platform:-unset}/${arch:-unset}" >&2 + exit 1 + ;; + esac +} + +android_target() { + case "$1" in + arm64-v8a) echo "aarch64-linux-android" ;; + x86_64) echo "x86_64-linux-android" ;; + *) return 1 ;; + esac +} + +build_android_abi() { + local abi="$1" + local target + target="$(android_target "$abi")" + local ndk_root="${ANDROID_NDK_HOME:-${ANDROID_NDK_ROOT:-}}" + if [[ -z "$ndk_root" ]]; then + echo "ANDROID_NDK_HOME must point at an Android NDK" >&2 + exit 1 + fi + local host_os + case "$(uname -s)" in + Darwin) host_os="darwin" ;; + Linux) host_os="linux" ;; + MINGW* | MSYS* | CYGWIN*) host_os="windows" ;; + *) + echo "Unsupported Android NDK host OS: $(uname -s)" >&2 + exit 1 + ;; + esac + local host_arch + case "$(uname -m)" in + arm64 | aarch64) host_arch="arm64" ;; + x86_64 | amd64) host_arch="x86_64" ;; + *) + echo "Unsupported Android NDK host architecture: $(uname -m)" >&2 + exit 1 + ;; + esac + local api="${BUZZ_ANDROID_MIN_API:-24}" + local clang_target + if [[ "$abi" == "arm64-v8a" ]]; then + clang_target="aarch64-linux-android${api}" + else + clang_target="x86_64-linux-android${api}" + fi + local prebuilt_root="${ndk_root}/toolchains/llvm/prebuilt" + local toolchain="${prebuilt_root}/${host_os}-${host_arch}" + local linker="${toolchain}/bin/${clang_target}-clang" + if [[ ! -x "$linker" ]]; then + local candidate + for candidate in "${prebuilt_root}/${host_os}-"*; do + if [[ -x "${candidate}/bin/${clang_target}-clang" ]]; then + toolchain="$candidate" + linker="${candidate}/bin/${clang_target}-clang" + break + fi + done + fi + if [[ ! -x "$linker" ]]; then + echo "Android linker not found: $linker" >&2 + exit 1 + fi + local source_libs="${cache_root}/android/jniLibs/${abi}" + local target_env + target_env="$(printf '%s' "$target" | tr '[:lower:]-' '[:upper:]_')" + local linker_var="CARGO_TARGET_${target_env}_LINKER" + local rustflags_var="CARGO_TARGET_${target_env}_RUSTFLAGS" + if ! rustup target list --installed | grep -Fxq "$target"; then + rustup target add "$target" + fi + env \ + SHERPA_ONNX_LIB_DIR="$source_libs" \ + "$linker_var=$linker" \ + "$rustflags_var=-L native=${source_libs}" \ + cargo rustc --release -p buzz-voice-mobile --no-default-features \ + --features shared --target "$target" --lib -- --crate-type cdylib + + local destination="${generated_root}/android/jniLibs/${abi}" + mkdir -p "$destination" + local shared_library="${repo_root}/target/${target}/release/libbuzz_voice_mobile.so" + if [[ ! -f "$shared_library" ]]; then + echo "Android cdylib was not produced for $target" >&2 + exit 1 + fi + cp "$shared_library" "${destination}/libbuzz_voice_mobile.so" + cp "${source_libs}/libsherpa-onnx-c-api.so" "$destination/" + cp "${source_libs}/libonnxruntime.so" "$destination/" +} + +build_android() { + fetch_native + build_android_abi "arm64-v8a" + build_android_abi "x86_64" +} + +case "${1:-}" in + fetch) fetch_native ;; + build-ios) build_ios ;; + build-ios-current) build_ios_current ;; + build-android) build_android ;; + build-all) + build_ios + build_android + ;; + *) usage ;; +esac diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 4dd1142801..ae26fc6ba8 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -84,6 +84,9 @@ run_unit_tests() { run_test_step "buzz-auth unit tests" \ cargo test -p buzz-auth --lib -- --nocapture + run_test_step "buzz-voice tests" \ + cargo test -p buzz-voice --lib -- --nocapture + # buzz-db migrator/lint unit tests (no infra): guard the embedded-migrator # invariant (exactly the consolidated 0001; cutover/backfill stays an operator # script, not startup state) and the tenant-scoping lints. The Postgres-backed