Skip to content

fix(asr): 修复 Windows 本地长语音只识别前段 #769

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Duyi-Wang wants to merge 1 commit into
base: beta
Choose a base branch
from
Open

fix(asr): 修复 Windows 本地长语音只识别前段 #769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 73 additions & 17 deletions openless-all/app/src-tauri/src/asr/local/foundry_provider.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ use crate::asr::RawTranscript;
#[cfg(target_os = "windows")]
use super::foundry_runtime::FoundryLocalRuntime;

/// Foundry Local Whisper 属于 Whisper 系模型，原生解码窗口约 30s。每次 SDK
/// 请求保持在窗口内，再由 OpenLess 合并分片文本，避免长听写只返回第一段。
const FOUNDRY_WHISPER_CHUNK_LIMIT_MS: u64 = 30_000;

pub struct FoundryLocalWhisperAsr {
#[cfg(target_os = "windows")]
runtime: Arc<FoundryLocalRuntime>,
Expand Down Expand Up @@ -70,6 +74,12 @@ impl FoundryLocalWhisperAsr {
self.language_hint.as_deref()
}

/// 当前缓冲音频时长（毫秒）。Coordinator 在 transcribe() 调用前读取，
/// 用来给 Foundry Local Whisper 计算动态超时。不消费缓冲。
pub fn buffer_duration_ms(&self) -> u64 {
pcm_duration_ms(&self.buffer.lock())
}

pub async fn transcribe(&self, audio_timeout: std::time::Duration) -> Result<RawTranscript> {
let cancel_generation = self.cancel_generation.load(Ordering::SeqCst);
let pcm = self.buffer.lock().clone();
Expand Down Expand Up @@ -108,26 +118,45 @@ impl FoundryLocalWhisperAsr {

#[cfg(target_os = "windows")]
{
let wav_file = TempWavFile::create(pcm)?;
let text = self
.runtime
.transcribe_audio_file(
&self.model_alias,
&self.runtime_source,
self.language_hint(),
wav_file.path(),
audio_timeout,
)
.await
.with_context(|| {
format!(
"transcribe audio file with Foundry Local Whisper model {}",
self.model_alias
let chunks = crate::asr::whisper::split_pcm_by_duration(
pcm,
Some(FOUNDRY_WHISPER_CHUNK_LIMIT_MS),
);
if chunks.len() > 1 {
log::info!(
"[foundry-asr] splitting {:.2}s audio into {} chunks (limit={}ms)",
duration_ms as f64 / 1000.0,
chunks.len(),
FOUNDRY_WHISPER_CHUNK_LIMIT_MS
);
}

let mut texts = Vec::with_capacity(chunks.len());
for (index, chunk) in chunks.iter().enumerate() {
let wav_file = TempWavFile::create(chunk)?;
let text = self
.runtime
.transcribe_audio_file(
&self.model_alias,
&self.runtime_source,
self.language_hint(),
wav_file.path(),
audio_timeout,
)
})?;
.await
.with_context(|| {
format!(
"transcribe Foundry Local Whisper chunk {}/{} with model {}",
index + 1,
chunks.len(),
self.model_alias
)
})?;
texts.push(trim_transcript_text(&text));
}

Ok(RawTranscript {
text: trim_transcript_text(&text),
text: crate::asr::whisper::join_transcript_chunks(&texts),
duration_ms,
})
}
Expand Down Expand Up @@ -293,6 +322,33 @@ mod tests {
assert_eq!(&wav[44..], &[0x01, 0x00, 0xff, 0x7f]);
}

#[test]
fn foundry_provider_splits_long_pcm_at_whisper_window() {
let pcm = vec![0u8; 32_000 * 65];
let chunks = crate::asr::whisper::split_pcm_by_duration(
&pcm,
Some(super::FOUNDRY_WHISPER_CHUNK_LIMIT_MS),
);

assert_eq!(chunks.len(), 3);
assert_eq!(chunks[0].len(), 32_000 * 30);
assert_eq!(chunks[1].len(), 32_000 * 30);
assert_eq!(chunks[2].len(), 32_000 * 5);
}

#[test]
fn foundry_provider_reports_buffer_duration_without_consuming() {
#[cfg(target_os = "windows")]
let (provider, _) = test_provider();
#[cfg(not(target_os = "windows"))]
let provider = test_provider();

provider.consume_pcm_chunk(&vec![0u8; 32_000]);

assert_eq!(provider.buffer_duration_ms(), 1000);
assert_eq!(provider.buffer.lock().len(), 32_000);
}

#[cfg(target_os = "windows")]
#[test]
fn foundry_provider_temp_wav_drop_removes_file() {
Expand Down
25 changes: 25 additions & 0 deletions openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,19 @@ impl SherpaOnnxAsr {
self.language_hint.as_deref()
}

/// 当前缓冲音频时长（毫秒）。Offline 读 PCM buffer；Online 读 worker 已接收
/// 的 PCM 字节数。不消费缓冲。
pub fn buffer_duration_ms(&self) -> u64 {
match &self.mode {
SherpaProviderMode::Offline { buffer } => pcm_duration_ms(&buffer.lock()),
SherpaProviderMode::Online { worker } => worker
.lock()
.as_ref()
.map(|worker| pcm_duration_ms_from_bytes(worker.audio_bytes.load(Ordering::SeqCst)))
.unwrap_or(0),
}
}

pub async fn transcribe(&self, audio_timeout: Duration) -> Result<RawTranscript> {
match &self.mode {
SherpaProviderMode::Offline { buffer } => {
Expand Down Expand Up @@ -390,6 +403,18 @@ mod tests {
}
}

#[test]
fn offline_buffer_duration_reports_16k_pcm_duration_without_consuming() {
let provider = make_provider();
provider.consume_pcm_chunk(&vec![0u8; 32_000]);

assert_eq!(provider.buffer_duration_ms(), 1000);
match &provider.mode {
SherpaProviderMode::Offline { buffer } => assert_eq!(buffer.lock().len(), 32_000),
SherpaProviderMode::Online { .. } => panic!("expected offline provider"),
}
}

#[tokio::test]
async fn empty_buffer_transcribe_returns_empty_transcript() {
let provider = make_provider();
Expand Down
4 changes: 2 additions & 2 deletions openless-all/app/src-tauri/src/asr/whisper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ fn pcm_duration_ms(pcm: &[u8]) -> u64 {
super::pcm::pcm_duration_ms(pcm)
}

fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: Option<u64>) -> Vec<&[u8]> {
pub(crate) fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: Option<u64>) -> Vec<&[u8]> {
let Some(max_chunk_duration_ms) = max_chunk_duration_ms else {
return vec![pcm];
};
Expand Down Expand Up @@ -328,7 +328,7 @@ fn transcription_url(base_url: &str) -> Result<String> {
Ok(url.to_string())
}

fn join_transcript_chunks(chunks: &[String]) -> String {
pub(crate) fn join_transcript_chunks(chunks: &[String]) -> String {
let mut joined = String::new();
for chunk in chunks.iter().map(|chunk| chunk.trim()) {
if chunk.is_empty() {
Expand Down
62 changes: 43 additions & 19 deletions openless-all/app/src-tauri/src/coordinator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1575,15 +1575,21 @@ impl Coordinator {
.map_err(|_| "重新转录超时".to_string())?
.map_err(|e| e.to_string())?,
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => local
.transcribe(foundry_audio_transcribe_timeout_duration())
.await
.map_err(|e| e.to_string())?,
ActiveAsr::FoundryLocalWhisper(local) => {
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
local
.transcribe(foundry_audio_transcribe_timeout(audio_secs))
.await
.map_err(|e| e.to_string())?
}
#[cfg(target_os = "windows")]
ActiveAsr::SherpaOnnxLocal(local) => local
.transcribe(sherpa_audio_transcribe_timeout_duration())
.await
.map_err(|e| e.to_string())?,
ActiveAsr::SherpaOnnxLocal(local) => {
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
local
.transcribe(sherpa_audio_transcribe_timeout(audio_secs))
.await
.map_err(|e| e.to_string())?
}
#[cfg(target_os = "macos")]
ActiveAsr::Local(local) => {
let dur =
Expand Down Expand Up @@ -2379,17 +2385,31 @@ mod tests {
assert!(!asr_transcribe_uses_global_timeout(&active_asr));
}

#[cfg(target_os = "windows")]
#[test]
fn foundry_audio_transcribe_timeout_is_separate_from_prepare() {
let timeout = foundry_audio_transcribe_timeout_duration();

fn foundry_timeout_floors_at_global_timeout_for_short_audio() {
assert_eq!(
timeout,
foundry_audio_transcribe_timeout(5.0),
std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS)
);
}

#[test]
fn foundry_timeout_scales_with_audio_duration() {
// 65s 录音：65 × 1.0 = 65，+20 = 85s。长音频不再撞 30s 墙。
assert_eq!(
foundry_audio_transcribe_timeout(65.0),
std::time::Duration::from_secs(85)
);
}

#[test]
fn sherpa_timeout_scales_with_audio_duration() {
assert_eq!(
sherpa_audio_transcribe_timeout(65.0),
std::time::Duration::from_secs(85)
);
}

#[test]
fn local_qwen_timeout_floors_at_global_timeout_for_short_audio() {
// 5s 录音：5 × 0.6 = 3, +10 = 13, max(30) = 30。短录音兜底。
Expand Down Expand Up @@ -3021,9 +3041,11 @@ const POST_SESSION_COOLDOWN_MS: u64 = 600;
/// 网络超时预算；只在 ASR 自身超时机制失效时作为最后的防线触发。
const COORDINATOR_GLOBAL_TIMEOUT_SECS: u64 = 30;

#[cfg(target_os = "windows")]
fn foundry_audio_transcribe_timeout_duration() -> std::time::Duration {
std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS)
fn foundry_audio_transcribe_timeout(audio_secs: f64) -> std::time::Duration {
let secs = ((audio_secs * 1.0).ceil() as u64)
.saturating_add(20)
.max(COORDINATOR_GLOBAL_TIMEOUT_SECS);
std::time::Duration::from_secs(secs)
}

/// 本地 Qwen3-ASR 的动态转写超时。固定 15 秒在长录音（≥ 30s）+ 慢机器
Expand All @@ -3050,9 +3072,11 @@ fn whisper_transcribe_timeout(audio_secs: f64) -> std::time::Duration {

/// sherpa-onnx offline batch 暂与 Foundry 同档；后续按 Windows 真机 CPU/模型
/// 实测结果再调整。
#[cfg(target_os = "windows")]
fn sherpa_audio_transcribe_timeout_duration() -> std::time::Duration {
std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS)
fn sherpa_audio_transcribe_timeout(audio_secs: f64) -> std::time::Duration {
let secs = ((audio_secs * 1.0).ceil() as u64)
.saturating_add(20)
.max(COORDINATOR_GLOBAL_TIMEOUT_SECS);
std::time::Duration::from_secs(secs)
}

pub(crate) fn validate_llm_endpoint(raw: &str) -> anyhow::Result<()> {
Expand Down
24 changes: 16 additions & 8 deletions openless-all/app/src-tauri/src/coordinator/dictation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2247,10 +2247,14 @@ pub(super) async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(foundry_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = foundry_audio_transcribe_timeout(audio_secs);
log::info!(
"[coord] Foundry Local Whisper transcribe: audio={:.2}s timeout={}s",
audio_secs,
timeout_duration.as_secs()
);
match local.transcribe(timeout_duration).await {
Ok(r) => {
schedule_foundry_local_asr_release(
inner,
Expand Down Expand Up @@ -2285,10 +2289,14 @@ pub(super) async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
#[cfg(target_os = "windows")]
ActiveAsr::SherpaOnnxLocal(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(sherpa_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = sherpa_audio_transcribe_timeout(audio_secs);
log::info!(
"[coord] sherpa-onnx transcribe: audio={:.2}s timeout={}s",
audio_secs,
timeout_duration.as_secs()
);
match local.transcribe(timeout_duration).await {
Ok(r) => {
schedule_sherpa_onnx_release(
inner,
Expand Down
38 changes: 22 additions & 16 deletions openless-all/app/src-tauri/src/coordinator/qa_session.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,10 +263,9 @@ pub(super) async fn transcribe_overlay_dictation_asr(
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(foundry_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = foundry_audio_transcribe_timeout(audio_secs);
match local.transcribe(timeout_duration).await {
Ok(raw) => {
schedule_foundry_local_asr_release(
_inner,
Expand All @@ -286,10 +285,9 @@ pub(super) async fn transcribe_overlay_dictation_asr(
#[cfg(target_os = "windows")]
ActiveAsr::SherpaOnnxLocal(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(sherpa_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = sherpa_audio_transcribe_timeout(audio_secs);
match local.transcribe(timeout_duration).await {
Ok(raw) => {
schedule_sherpa_onnx_release(
_inner,
Expand Down Expand Up @@ -826,10 +824,14 @@ pub(super) async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(foundry_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = foundry_audio_transcribe_timeout(audio_secs);
log::info!(
"[coord] QA Foundry Local Whisper transcribe: audio={:.2}s timeout={}s",
audio_secs,
timeout_duration.as_secs()
);
match local.transcribe(timeout_duration).await {
Ok(r) => {
schedule_foundry_local_asr_release(inner, AsrReleaseSession::Qa(qa_session_id));
r
Expand All @@ -852,10 +854,14 @@ pub(super) async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
#[cfg(target_os = "windows")]
ActiveAsr::SherpaOnnxLocal(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(sherpa_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = sherpa_audio_transcribe_timeout(audio_secs);
log::info!(
"[coord] QA sherpa-onnx transcribe: audio={:.2}s timeout={}s",
audio_secs,
timeout_duration.as_secs()
);
match local.transcribe(timeout_duration).await {
Ok(r) => {
schedule_sherpa_onnx_release(inner, AsrReleaseSession::Qa(qa_session_id));
r
Expand Down
Loading